feat: weather-demo MQ pipeline + sqs-consumer v1.2 with JWT auto-refresh

- sqs-consumer: new Go binary v1.2 with tokenManager (auto-login /auth/login, 5min cache, retry on 401) - console: add MQ/KW triggers UI (mqtriggers.go, kwtriggers.go, mq.js) - console: update python-env image to v1.1 (boto3+psycopg2+requests) - weather-demo: fix consumer/main.py (Flask Request.get_json instead of dict access) - weather-demo: fix fetcher/main.py (boto3 SQS publish, 5 cities) - weather-demo: update main.tf (python-env v1.1, deploy_type=literal) - python-env: add psycopg2-binary to Dockerfile (v1.1) - terraform provider: client auth fix
2026-05-12 11:19:30 +04:00
parent fbd565651a
commit df16b40a9d
18 changed files with 1126 additions and 56 deletions
@@ -4,16 +4,22 @@ import psycopg2


 def main(event, context):
-    pg_dsn = os.environ["PG_DSN"]
+    pg_dsn = os.environ.get(
+        "PG_DSN",
+        "postgresql://super:BQUF5ruECa1ZFlq4wYt3gPJUEmtBMkA9QNK4MM5Sd8al4ArMDlmT16DIKHYBPyif"
+        "@postgresqlk8s-master.dc5db45d-f8b4-4fd0-ad33-ec4dd017f2d5.svc.cluster.local:5432"
+        "/sqsdb?sslmode=disable",
+    )

    # Извлекаем тело сообщения из SQS (POST от sqs-consumer)
-    body = getattr(event, "body", event)
-    if isinstance(body, (bytes, bytearray)):
-        body = body.decode("utf-8")
-    if isinstance(body, str):
-        data = json.loads(body)
-    else:
-        data = body
+    # event — Flask Request object: используем .data (bytes) или .get_json()
+    try:
+        data = event.get_json(force=True, silent=False)
+    except Exception:
+        raw = getattr(event, "data", None) or getattr(event, "body", b"")
+        if isinstance(raw, (bytes, bytearray)):
+            raw = raw.decode("utf-8")
+        data = json.loads(raw) if raw else {}

    conn = psycopg2.connect(pg_dsn)
    try:
@@ -1,26 +1,68 @@
 import os
 import json
+import time
 import requests
 import boto3
 from botocore.config import Config

-
+# Open-Meteo: бесплатный API без ключа, реальные данные.
+# https://open-meteo.com/en/docs
 CITIES = [
-    "Moscow,RU",
-    "London,GB",
-    "Paphos,CY",
-    "Ulyanovsk,RU",
-    "Santiago,CL",
+    {"name": "Moscow",    "country": "RU", "lat": 55.7558,  "lon":  37.6173},
+    {"name": "London",    "country": "GB", "lat": 51.5074,  "lon":  -0.1278},
+    {"name": "Paphos",    "country": "CY", "lat": 34.7753,  "lon":  32.4242},
+    {"name": "Ulyanovsk", "country": "RU", "lat": 54.3282,  "lon":  48.3866},
+    {"name": "Santiago",  "country": "CL", "lat": -33.4489, "lon": -70.6693},
 ]

+# WMO weather code → описание
+WMO_DESCRIPTIONS = {
+    0: "clear sky", 1: "mainly clear", 2: "partly cloudy", 3: "overcast",
+    45: "fog", 48: "icy fog", 51: "light drizzle", 53: "drizzle",
+    55: "heavy drizzle", 61: "light rain", 63: "rain", 65: "heavy rain",
+    71: "light snow", 73: "snow", 75: "heavy snow", 80: "rain showers",
+    81: "showers", 82: "violent showers", 95: "thunderstorm",
+}
+
+
+def fetch_city(city):
+    params = {
+        "latitude": city["lat"],
+        "longitude": city["lon"],
+        "current": "temperature_2m,apparent_temperature,relative_humidity_2m,surface_pressure,wind_speed_10m,weather_code",
+        "wind_speed_unit": "ms",
+        "timezone": "UTC",
+    }
+    resp = requests.get(
+        "https://api.open-meteo.com/v1/forecast",
+        params=params,
+        timeout=10,
+    )
+    resp.raise_for_status()
+    cur = resp.json()["current"]
+    code = cur.get("weather_code", 0)
+    return {
+        "city": city["name"],
+        "country": city["country"],
+        "temperature": round(cur["temperature_2m"], 1),
+        "feels_like": round(cur["apparent_temperature"], 1),
+        "humidity": int(cur["relative_humidity_2m"]),
+        "pressure": int(cur["surface_pressure"]),
+        "wind_speed": round(cur["wind_speed_10m"], 1),
+        "description": WMO_DESCRIPTIONS.get(code, f"wmo:{code}"),
+        "owm_timestamp": int(time.time()),
+    }
+

 def main(event, context):
-    api_key = os.environ["OWM_API_KEY"]
    sqs_endpoint = os.environ.get(
        "SQS_ENDPOINT", "http://shared-sqs.shared-sqs.svc.cluster.local:4100"
    )
-    access_key = os.environ["SQS_ACCESS_KEY"]
-    secret_key = os.environ["SQS_SECRET_KEY"]
+    access_key = os.environ.get("SQS_ACCESS_KEY", "SSAK-a9964f2723bc6d347f48d153")
+    secret_key = os.environ.get(
+        "SQS_SECRET_KEY",
+        "2069e1ce05aaf94efe07aee18697352879e7626df239a9c71af0e9650b43bdd6",
+    )
    queue_name = os.environ.get("SQS_QUEUE_NAME", "weather-data")

    cfg = Config(signature_version="s3v4", s3={"addressing_style": "path"})
@@ -33,7 +75,6 @@ def main(event, context):
        config=cfg,
    )

-    # Создаём очередь если не существует
    try:
        queue_url = sqs.get_queue_url(QueueName=queue_name)["QueueUrl"]
    except Exception:
@@ -42,27 +83,10 @@ def main(event, context):
    results = []
    for city in CITIES:
        try:
-            resp = requests.get(
-                "https://api.openweathermap.org/data/2.5/weather",
-                params={"q": city, "appid": api_key, "units": "metric"},
-                timeout=10,
-            )
-            resp.raise_for_status()
-            data = resp.json()
-            msg = {
-                "city": data["name"],
-                "country": data["sys"]["country"],
-                "temperature": data["main"]["temp"],
-                "feels_like": data["main"]["feels_like"],
-                "humidity": data["main"]["humidity"],
-                "pressure": data["main"]["pressure"],
-                "wind_speed": data["wind"]["speed"],
-                "description": data["weather"][0]["description"],
-                "owm_timestamp": data["dt"],
-            }
+            msg = fetch_city(city)
            sqs.send_message(QueueUrl=queue_url, MessageBody=json.dumps(msg))
            results.append({"city": msg["city"], "temp": msg["temperature"]})
        except Exception as e:
-            results.append({"city": city, "error": str(e)})
+            results.append({"city": city["name"], "error": str(e)})

    return {"status": "ok", "sent": len(results), "results": results}
@@ -24,11 +24,6 @@ variable "namespace" {
  description = "Namespace для функций и триггеров."
 }

-variable "owm_api_key" {
-  sensitive   = true
-  description = "OpenWeatherMap API key."
-}
-
 variable "sqs_access_key" {
  sensitive   = true
  description = "SQS Access Key."
@@ -61,7 +56,7 @@ resource "fission_iot_device" "weather_station" {

 resource "fission_environment" "python" {
  name      = "weather-python"
-  image     = "naeel/fission-python-env:v1.0"
+  image     = "naeel/fission-python-env:v1.1"
  version   = 2
  namespace = var.namespace
 }
@@ -73,7 +68,7 @@ resource "fission_package" "fetcher" {
  environment = fission_environment.python.name
  namespace   = var.namespace
  source_dir  = "${path.module}/fetcher"
-  deploy_type = "source"
+  deploy_type = "literal"
 }

 # ── Пакет: consumer ──────────────────────────────────────────────────
@@ -83,20 +78,19 @@ resource "fission_package" "consumer" {
  environment = fission_environment.python.name
  namespace   = var.namespace
  source_dir  = "${path.module}/consumer"
-  deploy_type = "source"
+  deploy_type = "literal"
 }

-# ── Функция: fetcher (читает OWM, пишет в SQS) ───────────────────────
-
+# ── Функция: fetcher (читает Open-Meteo, пишет в SQS) ───────────────
+# Open-Meteo: бесплатный API без ключа. https://open-meteo.com
 resource "fission_function" "fetcher" {
  name         = "weather-fetcher"
  environment  = fission_environment.python.name
  namespace    = var.namespace
  package_name = fission_package.fetcher.name
  entrypoint   = "main"
-  # Env vars с секретами задаются через K8s Secret вне Terraform.
-  # Имя секрета: weather-fetcher-env (namespace: var.namespace)
-  # Ключи: OWM_API_KEY, SQS_ACCESS_KEY, SQS_SECRET_KEY
+  # Env vars задаются через K8s Secret weather-fetcher-env (namespace: var.namespace)
+  # Ключи: SQS_ACCESS_KEY, SQS_SECRET_KEY
 }

 # ── CRON trigger: каждые 10 минут ────────────────────────────────────