feat: weather-demo MQ pipeline + sqs-consumer v1.2 with JWT auto-refresh

- sqs-consumer: new Go binary v1.2 with tokenManager (auto-login /auth/login, 5min cache, retry on 401) - console: add MQ/KW triggers UI (mqtriggers.go, kwtriggers.go, mq.js) - console: update python-env image to v1.1 (boto3+psycopg2+requests) - weather-demo: fix consumer/main.py (Flask Request.get_json instead of dict access) - weather-demo: fix fetcher/main.py (boto3 SQS publish, 5 cities) - weather-demo: update main.tf (python-env v1.1, deploy_type=literal) - python-env: add psycopg2-binary to Dockerfile (v1.1) - terraform provider: client auth fix
2026-05-12 11:19:30 +04:00
parent fbd565651a
commit df16b40a9d
18 changed files with 1126 additions and 56 deletions
@@ -0,0 +1,241 @@
+package api
+
+// kwtriggers.go — CRUD хендлеры для KubernetesWatchTrigger (Fission KW Trigger).
+//
+// РЕШЕНИЕ ПО АРХИТЕКТУРЕ (2026-05-11):
+//   KubernetesWatchTrigger позволяет вызывать функцию при изменении K8s объектов.
+//   spec.type      — тип ресурса: Pod, Service, Deployment, ConfigMap, и т.д.
+//   spec.namespace — namespace для слежения (по умолчанию = namespace пользователя)
+//   spec.labelselector — label selector в формате "key=value,key2=value2"
+//   spec.functionref — ссылка на функцию
+//
+// ОСОБЕННОСТИ:
+//   - Fission kubewatcher компонент должен быть задеплоен.
+//   - namespace в spec — это WATCHED namespace (не namespace триггера).
+//     Для безопасности ограничиваем: только namespace пользователя или пустое (тогда = userNS).
+//   - labelselector опционален, "" = смотрим на все ресурсы типа resourceType в namespace.
+//
+// ОШИБКИ В ПРОЦЕССЕ:
+//   - (нет, первая реализация)
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"fission-console/internal/fission"
+	"fission-console/internal/model"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+// validKWResourceTypes — поддерживаемые типы ресурсов для слежения.
+// Расширяемо — это не ограничение CRD, просто UI-валидация.
+var validKWResourceTypes = map[string]struct{}{
+	"pod":         {},
+	"service":     {},
+	"deployment":  {},
+	"configmap":   {},
+	"secret":      {},
+	"namespace":   {},
+	"replicaset":  {},
+	"statefulset": {},
+	"daemonset":   {},
+	"job":         {},
+}
+
+func (s *Server) handleKWTriggersRoot(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case http.MethodGet:
+		s.handleList(fission.KWTrigGVR)(w, r)
+	case http.MethodPost:
+		s.handleCreateKWTrigger(w, r)
+	default:
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *Server) handleKWTriggersAction(w http.ResponseWriter, r *http.Request) {
+	path := strings.TrimPrefix(r.URL.Path, "/console/api/kwtriggers/")
+	path = strings.TrimPrefix(path, "/api/kwtriggers/")
+	name := strings.Trim(path, "/")
+	if name == "" || strings.Contains(name, "/") {
+		http.NotFound(w, r)
+		return
+	}
+
+	switch r.Method {
+	case http.MethodGet:
+		s.handleGetKWTrigger(w, r, name)
+	case http.MethodDelete:
+		s.handleDeleteKWTrigger(w, r, name)
+	default:
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *Server) handleCreateKWTrigger(w http.ResponseWriter, r *http.Request) {
+	var req model.CreateKWTriggerRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("decode request: %v", err))
+		return
+	}
+
+	ns := s.userNS(r)
+
+	// Если namespace не задан — используем namespace пользователя
+	if strings.TrimSpace(req.Namespace) == "" {
+		req.Namespace = ns
+	}
+
+	// Безопасность: нельзя смотреть за чужим namespace
+	if req.Namespace != ns {
+		writeJSONError(w, http.StatusForbidden, "can only watch your own namespace")
+		return
+	}
+
+	if err := validateKWTriggerRequest(req); err != nil {
+		writeJSONError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+	defer cancel()
+
+	// Проверяем что функция существует
+	if _, err := s.dyn.Resource(fission.FunctionGVR).Namespace(ns).Get(ctx, req.FunctionName, metav1.GetOptions{}); err != nil {
+		if apierrors.IsNotFound(err) {
+			writeJSONError(w, http.StatusBadRequest, fmt.Sprintf("function %q not found", req.FunctionName))
+			return
+		}
+		writeJSONError(w, http.StatusBadGateway, fmt.Sprintf("get function: %v", err))
+		return
+	}
+
+	obj := buildKWTriggerObject(ns, req)
+	created, err := s.dyn.Resource(fission.KWTrigGVR).Namespace(ns).Create(ctx, obj, metav1.CreateOptions{})
+	if err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			writeJSONError(w, http.StatusConflict, fmt.Sprintf("kwtrigger %q already exists", req.Name))
+			return
+		}
+		writeJSONError(w, http.StatusBadGateway, fmt.Sprintf("create kwtrigger: %v", err))
+		return
+	}
+
+	writeAnyJSON(w, http.StatusCreated, kwTriggerResponse(created))
+}
+
+func (s *Server) handleGetKWTrigger(w http.ResponseWriter, r *http.Request, name string) {
+	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+	defer cancel()
+
+	obj, err := s.dyn.Resource(fission.KWTrigGVR).Namespace(s.userNS(r)).Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		status := http.StatusBadGateway
+		if apierrors.IsNotFound(err) {
+			status = http.StatusNotFound
+		}
+		writeJSONError(w, status, fmt.Sprintf("get kwtrigger %q: %v", name, err))
+		return
+	}
+
+	writeAnyJSON(w, http.StatusOK, kwTriggerResponse(obj))
+}
+
+func (s *Server) handleDeleteKWTrigger(w http.ResponseWriter, r *http.Request, name string) {
+	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+	defer cancel()
+
+	if err := s.dyn.Resource(fission.KWTrigGVR).Namespace(s.userNS(r)).Delete(ctx, name, metav1.DeleteOptions{}); err != nil {
+		status := http.StatusBadGateway
+		if apierrors.IsNotFound(err) {
+			status = http.StatusNotFound
+		}
+		writeJSONError(w, status, fmt.Sprintf("delete kwtrigger %q: %v", name, err))
+		return
+	}
+
+	writeAnyJSON(w, http.StatusOK, map[string]any{"deleted": true, "name": name})
+}
+
+// --- Вспомогательные функции ---
+
+func validateKWTriggerRequest(req model.CreateKWTriggerRequest) error {
+	if strings.TrimSpace(req.Name) == "" {
+		return fmt.Errorf("name is required")
+	}
+	if strings.TrimSpace(req.FunctionName) == "" {
+		return fmt.Errorf("functionName is required")
+	}
+	rt := strings.ToLower(strings.TrimSpace(req.ResourceType))
+	if _, ok := validKWResourceTypes[rt]; !ok {
+		return fmt.Errorf("resourceType must be one of: Pod, Service, Deployment, ConfigMap, Secret, Namespace, ReplicaSet, StatefulSet, DaemonSet, Job")
+	}
+	return nil
+}
+
+func buildKWTriggerObject(ns string, req model.CreateKWTriggerRequest) *unstructured.Unstructured {
+	// Fission ожидает capitalize: Pod, Service, Deployment
+	resourceType := capitalize(strings.TrimSpace(req.ResourceType))
+
+	spec := map[string]any{
+		"type":      resourceType,
+		"namespace": req.Namespace,
+		"functionref": map[string]any{
+			"type": "name",
+			"name": req.FunctionName,
+		},
+	}
+	if req.LabelSelector != "" {
+		spec["labelselector"] = req.LabelSelector
+	}
+
+	return &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": "fission.io/v1",
+		"kind":       "KubernetesWatchTrigger",
+		"metadata": map[string]any{
+			"name":      req.Name,
+			"namespace": ns,
+		},
+		"spec": spec,
+	}}
+}
+
+func kwTriggerResponse(obj *unstructured.Unstructured) map[string]any {
+	spec, _ := obj.Object["spec"].(map[string]any)
+	if spec == nil {
+		spec = map[string]any{}
+	}
+	fnref, _ := spec["functionref"].(map[string]any)
+	fnName := ""
+	if fnref != nil {
+		fnName, _ = fnref["name"].(string)
+	}
+	return map[string]any{
+		"metadata": map[string]any{
+			"name":      obj.GetName(),
+			"namespace": obj.GetNamespace(),
+		},
+		"spec": map[string]any{
+			"resourceType":  spec["type"],
+			"namespace":     spec["namespace"],
+			"labelSelector": spec["labelselector"],
+			"functionName":  fnName,
+		},
+	}
+}
+
+// capitalize приводит первый символ к верхнему регистру, остальное без изменений.
+// "pod" → "Pod", "deployment" → "Deployment"
+func capitalize(s string) string {
+	if s == "" {
+		return s
+	}
+	return strings.ToUpper(s[:1]) + strings.ToLower(s[1:])
+}
@@ -0,0 +1,286 @@
+package api
+
+// mqtriggers.go — CRUD для MQ-триггеров через K8s Deployment + Secret.
+//
+// АРХИТЕКТУРА (2026-05-11):
+//   Вместо Fission MessageQueueTrigger CRD (требует mqtrigger компонент Kafka/NATS)
+//   Console деплоит собственный sqs-consumer Deployment в namespace пользователя.
+//
+//   При CREATE:
+//     1. Создаём K8s Secret (sqs-mq-<name>) с SQS credentials
+//     2. Создаём K8s Deployment (mq-<name>) с образом naeel/sqs-consumer:v1.0
+//        FUNCTION_URL = http://router.fission.svc.cluster.local/<functionName>
+//     Лейблы: app.kubernetes.io/managed-by=fission-console, component=mq-trigger
+//
+//   При LIST: deployments -n <ns> -l component=mq-trigger
+//   При DELETE: удаляем Deployment + Secret
+//
+// ИЗМЕНЕНИЯ:
+//   v1 — использовал Fission MQ CRD (компонент отсутствует в кластере)
+//   v2 — K8s Deployment + наш sqs-consumer образ
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"fission-console/internal/model"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	sqsConsumerImage   = "naeel/sqs-consumer:v1.0"
+	sqsDefaultEndpoint = "http://shared-sqs.shared-sqs.svc.cluster.local:4100"
+	fissionRouterBase  = "http://router.fission.svc.cluster.local"
+	mqTriggerLabelKey  = "component"
+	mqTriggerLabelVal  = "mq-trigger"
+	mqManagedByLabel   = "app.kubernetes.io/managed-by"
+	mqManagedByVal     = "fission-console"
+)
+
+func mqSecretName(name string) string { return "sqs-mq-" + name }
+func mqDeployName(name string) string { return "mq-" + name }
+
+// ── HTTP хендлеры ────────────────────────────────────────────────────
+
+func (s *Server) handleMQTriggersRoot(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case http.MethodGet:
+		s.handleListMQTriggers(w, r)
+	case http.MethodPost:
+		s.handleCreateMQTrigger(w, r)
+	default:
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *Server) handleMQTriggersAction(w http.ResponseWriter, r *http.Request) {
+	path := strings.TrimPrefix(r.URL.Path, "/console/api/mqtriggers/")
+	name := strings.Trim(path, "/")
+	if name == "" || strings.Contains(name, "/") {
+		http.NotFound(w, r)
+		return
+	}
+	switch r.Method {
+	case http.MethodDelete:
+		s.handleDeleteMQTrigger(w, r, name)
+	default:
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// ── LIST ────────────────────────────────────────────────────────────
+
+func (s *Server) handleListMQTriggers(w http.ResponseWriter, r *http.Request) {
+	ns := s.userNS(r)
+	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+	defer cancel()
+
+	labelSel := fmt.Sprintf("%s=%s,%s=%s", mqManagedByLabel, mqManagedByVal, mqTriggerLabelKey, mqTriggerLabelVal)
+	deployList, err := s.kube.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{LabelSelector: labelSel})
+	if err != nil {
+		writeJSONError(w, http.StatusBadGateway, fmt.Sprintf("list mq deployments: %v", err))
+		return
+	}
+
+	items := make([]map[string]any, 0, len(deployList.Items))
+	for i := range deployList.Items {
+		items = append(items, mqDeployToResponse(&deployList.Items[i]))
+	}
+	writeAnyJSON(w, http.StatusOK, map[string]any{"items": items})
+}
+
+// ── CREATE ──────────────────────────────────────────────────────────
+
+func (s *Server) handleCreateMQTrigger(w http.ResponseWriter, r *http.Request) {
+	var req model.CreateMQTriggerRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeJSONError(w, http.StatusBadRequest, "decode request: "+err.Error())
+		return
+	}
+	if err := validateMQRequest(req); err != nil {
+		writeJSONError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+
+	ns := s.userNS(r)
+	ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
+	defer cancel()
+
+	endpoint := strings.TrimSpace(req.SqsEndpoint)
+	if endpoint == "" {
+		endpoint = sqsDefaultEndpoint
+	}
+	functionURL := fissionRouterBase + "/" + req.FunctionName
+	secretName := mqSecretName(req.Name)
+	deployName := mqDeployName(req.Name)
+
+	// 1. Secret с SQS credentials
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: ns,
+			Labels: map[string]string{
+				mqManagedByLabel:  mqManagedByVal,
+				mqTriggerLabelKey: mqTriggerLabelVal,
+				"mq-trigger-name": req.Name,
+			},
+		},
+		StringData: map[string]string{
+			"SQS_ACCESS_KEY": req.AccessKey,
+			"SQS_SECRET_KEY": req.SecretKey,
+			"SQS_ENDPOINT":   endpoint,
+		},
+	}
+	if _, err := s.kube.CoreV1().Secrets(ns).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			writeJSONError(w, http.StatusConflict, fmt.Sprintf("mq trigger %q already exists", req.Name))
+			return
+		}
+		writeJSONError(w, http.StatusBadGateway, "create secret: "+err.Error())
+		return
+	}
+
+	// 2. Deployment (sqs-consumer)
+	replicas := int32(1)
+	deploy := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      deployName,
+			Namespace: ns,
+			Labels: map[string]string{
+				mqManagedByLabel:  mqManagedByVal,
+				mqTriggerLabelKey: mqTriggerLabelVal,
+				"mq-trigger-name": req.Name,
+			},
+			Annotations: map[string]string{
+				"fission-console/mq-trigger-name": req.Name,
+				"fission-console/function":        req.FunctionName,
+				"fission-console/queue":           req.Queue,
+				"fission-console/sqs-endpoint":    endpoint,
+			},
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"mq-trigger-name": req.Name},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"mq-trigger-name": req.Name,
+						mqTriggerLabelKey: mqTriggerLabelVal,
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{{
+						Name:            "sqs-consumer",
+						Image:           sqsConsumerImage,
+						ImagePullPolicy: corev1.PullAlways,
+						Env: []corev1.EnvVar{
+							{Name: "SQS_QUEUE_NAME", Value: req.Queue},
+							{Name: "SQS_REGION", Value: "us-east-1"},
+							{Name: "FUNCTION_URL", Value: functionURL},
+							{Name: "POLL_INTERVAL", Value: "5"},
+							{Name: "MAX_MESSAGES", Value: "1"},
+							{Name: "MAX_RETRIES", Value: "3"},
+						},
+						EnvFrom: []corev1.EnvFromSource{{
+							SecretRef: &corev1.SecretEnvSource{
+								LocalObjectReference: corev1.LocalObjectReference{Name: secretName},
+							},
+						}},
+						Resources: corev1.ResourceRequirements{
+							Limits: corev1.ResourceList{
+								corev1.ResourceCPU:    resource.MustParse("50m"),
+								corev1.ResourceMemory: resource.MustParse("32Mi"),
+							},
+							Requests: corev1.ResourceList{
+								corev1.ResourceCPU:    resource.MustParse("10m"),
+								corev1.ResourceMemory: resource.MustParse("16Mi"),
+							},
+						},
+					}},
+				},
+			},
+		},
+	}
+
+	created, err := s.kube.AppsV1().Deployments(ns).Create(ctx, deploy, metav1.CreateOptions{})
+	if err != nil {
+		_ = s.kube.CoreV1().Secrets(ns).Delete(ctx, secretName, metav1.DeleteOptions{})
+		writeJSONError(w, http.StatusBadGateway, "create deployment: "+err.Error())
+		return
+	}
+
+	writeAnyJSON(w, http.StatusCreated, mqDeployToResponse(created))
+}
+
+// ── DELETE ──────────────────────────────────────────────────────────
+
+func (s *Server) handleDeleteMQTrigger(w http.ResponseWriter, r *http.Request, name string) {
+	ns := s.userNS(r)
+	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+	defer cancel()
+
+	dErr := s.kube.AppsV1().Deployments(ns).Delete(ctx, mqDeployName(name), metav1.DeleteOptions{})
+	sErr := s.kube.CoreV1().Secrets(ns).Delete(ctx, mqSecretName(name), metav1.DeleteOptions{})
+
+	if dErr != nil && !apierrors.IsNotFound(dErr) {
+		writeJSONError(w, http.StatusBadGateway, "delete deployment: "+dErr.Error())
+		return
+	}
+	if sErr != nil && !apierrors.IsNotFound(sErr) {
+		writeJSONError(w, http.StatusBadGateway, "delete secret: "+sErr.Error())
+		return
+	}
+	writeAnyJSON(w, http.StatusOK, map[string]any{"deleted": true, "name": name})
+}
+
+// ── Вспомогательные ─────────────────────────────────────────────────
+
+func validateMQRequest(req model.CreateMQTriggerRequest) error {
+	if strings.TrimSpace(req.Name) == "" {
+		return fmt.Errorf("name is required")
+	}
+	if strings.TrimSpace(req.FunctionName) == "" {
+		return fmt.Errorf("functionName is required")
+	}
+	if strings.TrimSpace(req.Queue) == "" {
+		return fmt.Errorf("queue is required")
+	}
+	if strings.TrimSpace(req.AccessKey) == "" {
+		return fmt.Errorf("accessKey is required")
+	}
+	if strings.TrimSpace(req.SecretKey) == "" {
+		return fmt.Errorf("secretKey is required")
+	}
+	return nil
+}
+
+func mqDeployToResponse(d *appsv1.Deployment) map[string]any {
+	ann := d.Annotations
+	if ann == nil {
+		ann = map[string]string{}
+	}
+	triggerName := ann["fission-console/mq-trigger-name"]
+	if triggerName == "" {
+		triggerName = strings.TrimPrefix(d.Name, "mq-")
+	}
+	return map[string]any{
+		"name":         triggerName,
+		"deployName":   d.Name,
+		"functionName": ann["fission-console/function"],
+		"queue":        ann["fission-console/queue"],
+		"sqsEndpoint":  ann["fission-console/sqs-endpoint"],
+		"ready":        d.Status.ReadyReplicas > 0,
+		"replicas":     d.Status.ReadyReplicas,
+	}
+}
@@ -169,6 +169,10 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) {
 	mux.HandleFunc("/console/api/httptriggers", auth(s.handleList(fission.HTTPTrigGVR)))
 	mux.HandleFunc("/console/api/timetriggers", auth(s.handleTimeTriggersRoot))
 	mux.HandleFunc("/console/api/timetriggers/", auth(s.handleTimeTriggersAction))
+	mux.HandleFunc("/console/api/mqtriggers", auth(s.handleMQTriggersRoot))
+	mux.HandleFunc("/console/api/mqtriggers/", auth(s.handleMQTriggersAction))
+	mux.HandleFunc("/console/api/kwtriggers", auth(s.handleKWTriggersRoot))
+	mux.HandleFunc("/console/api/kwtriggers/", auth(s.handleKWTriggersAction))
 	mux.HandleFunc("/console/api/ns/status", auth(s.handleNSStatus))
 	mux.HandleFunc("/console/api/ns/debug", auth(s.handleNSDebug))
 	mux.HandleFunc("/console/api/stats/dashboard-url", auth(s.handleStatsDashboard))