From 0adb379a545d4a94e68ade6e4f8c7e0c5d5e70c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Tue, 21 Jul 2026 18:32:35 +0400 Subject: [PATCH] =?UTF-8?q?feat:=20IoT=20demo=20final=20=E2=80=94=206=20se?= =?UTF-8?q?rvices=20(Kafka+Redis+MongoDB+2Flask+NodeJS),=20removed=20broke?= =?UTF-8?q?n=20CH/Superset/AKHQ?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DEV_STAND/IOT_KAFKA_DEMO/clickhouse.tf | 66 ------------------------ DEV_STAND/IOT_KAFKA_DEMO/consumer.tf | 25 +++++---- DEV_STAND/IOT_KAFKA_DEMO/dashboard.tf | 43 +++++++++++++++ DEV_STAND/IOT_KAFKA_DEMO/locals.tf | 40 +++++--------- DEV_STAND/IOT_KAFKA_DEMO/superset.tf | 14 ----- apps/iot-demo/HISTORY/09-final-deploy.md | 58 +++++++++++++++++++++ 6 files changed, 129 insertions(+), 117 deletions(-) delete mode 100644 DEV_STAND/IOT_KAFKA_DEMO/clickhouse.tf create mode 100644 DEV_STAND/IOT_KAFKA_DEMO/dashboard.tf delete mode 100644 DEV_STAND/IOT_KAFKA_DEMO/superset.tf create mode 100644 apps/iot-demo/HISTORY/09-final-deploy.md diff --git a/DEV_STAND/IOT_KAFKA_DEMO/clickhouse.tf b/DEV_STAND/IOT_KAFKA_DEMO/clickhouse.tf deleted file mode 100644 index 720b6df..0000000 --- a/DEV_STAND/IOT_KAFKA_DEMO/clickhouse.tf +++ /dev/null @@ -1,66 +0,0 @@ -# ============================================================================= -# ClickHouse — аналитическая БД -# ============================================================================= -resource "nubes_clickhouse" "main" { - resource_name = local.ch_resource_name - - startup_configuration = { - resource_realm = var.realm - } - - cluster_configuration = { - cpu = local.ch_cpu - memory = local.ch_memory - disk = local.ch_disk - replicas = local.ch_replicas - shards = local.ch_shards - } - - access_configuration = { - master_ip_space = "no-needed" - master_access_list = jsonencode(["10.0.0.0/8"]) - } - - clickhouse_configuration = { - version = local.ch_version - } - - clickhouse_keeper_configuration = { - cpu = 500 - memory = 512 - replicas = 1 - } - - backup_configuration = { - s3_uid = var.s3_name - retain = 7 - schedule = "0 0 * * *" - } - - autoscale_configuration = { - enabled = false - percent = 10 - quota = 100 - schedule = 0 - } - - operation_timeout = local.ch_timeout -} - -# ----------------------------------------------------------------------------- -# ClickHouse Database -# ----------------------------------------------------------------------------- -resource "nubes_clickhouse_database" "iot" { - clickhouse_id = nubes_clickhouse.main.id - db_name = local.ch_db_name -} - -# ----------------------------------------------------------------------------- -# ClickHouse User -# ----------------------------------------------------------------------------- -resource "nubes_clickhouse_user" "writer" { - clickhouse_id = nubes_clickhouse.main.id - username = local.ch_username - db_name = nubes_clickhouse_database.iot.db_name - access_hosts = "0.0.0.0/0" -} diff --git a/DEV_STAND/IOT_KAFKA_DEMO/consumer.tf b/DEV_STAND/IOT_KAFKA_DEMO/consumer.tf index be59d4e..79d20f1 100644 --- a/DEV_STAND/IOT_KAFKA_DEMO/consumer.tf +++ b/DEV_STAND/IOT_KAFKA_DEMO/consumer.tf @@ -1,5 +1,5 @@ # ============================================================================= -# Flask Consumer — читает Kafka, пишет в ClickHouse +# Flask Consumer — читает Kafka, пишет в Redis + MongoDB # ============================================================================= locals { consumer_kafka_secrets = jsondecode(nubes_kafka.main.vault_secrets["users"]) @@ -8,9 +8,14 @@ locals { consumer_kafka_ca_crt = try(nonsensitive(local.consumer_kafka_secrets["ca"]["ca.crt"]), "") consumer_kafka_user_crt = try(nonsensitive(local.consumer_kafka_secrets["iot-app"]["user.crt"]), "") consumer_kafka_user_key = try(nonsensitive(local.consumer_kafka_secrets["iot-app"]["user.key"]), "") - consumer_ch_secrets = try(jsondecode(nubes_clickhouse.main.vault_secrets["users"]), tomap({})) - consumer_ch_host = try(nubes_clickhouse.main.state_out_flat["internalMaster"], "") - consumer_ch_password = try(nonsensitive(lookup(local.consumer_ch_secrets, "iot_writer", tomap({})).password), "") + + consumer_redis_host = try(nubes_redis.main.state_out_flat["internalConnect.master"], "") + consumer_redis_secrets = try(jsondecode(nubes_redis.main.vault_secrets["users"]), tomap({})) + consumer_redis_password = try(nonsensitive(lookup(local.consumer_redis_secrets, "default", tomap({})).password), "") + + consumer_mongo_host = try(nubes_mongodb.main.state_out_flat["internalConnect"], "") + consumer_mongo_secrets = try(jsondecode(nubes_mongodb.main.vault_secrets["users"]), tomap({})) + consumer_mongo_password = try(nonsensitive(lookup(local.consumer_mongo_secrets, "admin", tomap({})).password), "") } resource "nubes_flask" "consumer" { @@ -48,12 +53,12 @@ resource "nubes_flask" "consumer" { KAFKA_USER_CRT = local.consumer_kafka_user_crt KAFKA_USER_KEY = local.consumer_kafka_user_key KAFKA_GROUP_ID = "iot-consumer-group" - CH_HOST = local.consumer_ch_host - CH_PORT = local.ch_http_port - CH_USER = local.ch_username - CH_PASSWORD = local.consumer_ch_password - CH_DATABASE = local.ch_db_name + REDIS_HOST = local.consumer_redis_host + REDIS_PORT = "6379" + REDIS_PASSWORD = local.consumer_redis_password + MONGO_URI = "mongodb://admin:${local.consumer_mongo_password}@${local.consumer_mongo_host}:27017/iot?authSource=admin" + MONGO_DB = "iot" }) - depends_on = [nubes_kafka.main, nubes_clickhouse.main] + depends_on = [nubes_kafka.main, nubes_redis.main, nubes_mongodb.main] } diff --git a/DEV_STAND/IOT_KAFKA_DEMO/dashboard.tf b/DEV_STAND/IOT_KAFKA_DEMO/dashboard.tf new file mode 100644 index 0000000..247d2d1 --- /dev/null +++ b/DEV_STAND/IOT_KAFKA_DEMO/dashboard.tf @@ -0,0 +1,43 @@ +# ============================================================================= +# Node.js Dashboard — читает Redis, показывает графики Chart.js +# ============================================================================= +locals { + dashboard_redis_host = try(nubes_redis.main.state_out_flat["internalConnect.master"], "") + dashboard_redis_secrets = try(jsondecode(nubes_redis.main.vault_secrets["users"]), tomap({})) + dashboard_redis_password = try(nonsensitive(lookup(local.dashboard_redis_secrets, "default", tomap({})).password), "") +} + +resource "nubes_nodejs" "dashboard" { + resource_name = local.dashboard_resource_name + + startup_configuration = { + resource_realm = var.realm + } + + cluster_configuration = { + cpu = local.dashboard_cpu + memory = local.dashboard_memory + replicas = local.dashboard_replicas + } + + access_configuration = { + domain = local.dashboard_domain + } + + app_configuration = { + version = "22" + git_path = local.dashboard_git_path + health_path = "/" + } + + git_revision = local.dashboard_git_revision + operation_timeout = local.dashboard_timeout + + json_env = jsonencode({ + REDIS_HOST = local.dashboard_redis_host + REDIS_PORT = "6379" + REDIS_PASSWORD = local.dashboard_redis_password + }) + + depends_on = [nubes_redis.main] +} diff --git a/DEV_STAND/IOT_KAFKA_DEMO/locals.tf b/DEV_STAND/IOT_KAFKA_DEMO/locals.tf index f15bb8e..be770aa 100644 --- a/DEV_STAND/IOT_KAFKA_DEMO/locals.tf +++ b/DEV_STAND/IOT_KAFKA_DEMO/locals.tf @@ -38,26 +38,6 @@ locals { ch_version = "24.8" ch_timeout = "15m" - # ClickHouse — database - ch_db_name = "iot" - - # ClickHouse — user - ch_username = "iot_writer" - ch_role = "ddl_user" - - # ═══════════════════════════════════════════════════════════════════════════ - # Superset - # ═══════════════════════════════════════════════════════════════════════════ - - superset_resource_name = "iot-superset" - superset_domain = "iotsuperset" - superset_cpu = 500 - superset_memory = 1024 - superset_disk = 5 - superset_instances = 1 - superset_email = "admin" - superset_timeout = "15m" - # ═══════════════════════════════════════════════════════════════════════════ # Flask — Producer # ═══════════════════════════════════════════════════════════════════════════ @@ -84,13 +64,6 @@ locals { consumer_replicas = 1 consumer_timeout = "11m" - # ═══════════════════════════════════════════════════════════════════════════ - # Общие - # ═══════════════════════════════════════════════════════════════════════════ - - kafka_port = "9092" - ch_http_port = "8123" - # ═══════════════════════════════════════════════════════════════════════════ # Redis # ═══════════════════════════════════════════════════════════════════════════ @@ -112,4 +85,17 @@ locals { mongo_disk = 5 mongo_replicas = 1 mongo_timeout = "11m" + + # ═══════════════════════════════════════════════════════════════════════════ + # Node.js Dashboard + # ═══════════════════════════════════════════════════════════════════════════ + + dashboard_resource_name = "iot-dashboard" + dashboard_domain = "iotdash" + dashboard_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-dashboard.git" + dashboard_git_revision = "master" + dashboard_cpu = 300 + dashboard_memory = 256 + dashboard_replicas = 1 + dashboard_timeout = "11m" } diff --git a/DEV_STAND/IOT_KAFKA_DEMO/superset.tf b/DEV_STAND/IOT_KAFKA_DEMO/superset.tf deleted file mode 100644 index 31d926f..0000000 --- a/DEV_STAND/IOT_KAFKA_DEMO/superset.tf +++ /dev/null @@ -1,14 +0,0 @@ -# ============================================================================= -# Superset — дашборды -# ============================================================================= -resource "nubes_superset" "dash" { - resource_name = local.superset_resource_name - resource_realm = var.realm - domain = local.superset_domain - resource_c_p_u = local.superset_cpu - resource_memory = local.superset_memory - resource_disk = local.superset_disk - resource_instances = local.superset_instances - emails = local.superset_email - operation_timeout = local.superset_timeout -} diff --git a/apps/iot-demo/HISTORY/09-final-deploy.md b/apps/iot-demo/HISTORY/09-final-deploy.md new file mode 100644 index 0000000..4626d0c --- /dev/null +++ b/apps/iot-demo/HISTORY/09-final-deploy.md @@ -0,0 +1,58 @@ +# Шаг 9 — DEV стенд: все 6 сервисов развёрнуты + +**Дата:** 2026-07-21 + +## Финальный состав (6 сервисов) + +| # | Ресурс Terraform | Имя | Роль | +|---|-----------------|------|------| +| 1 | `nubes_kafka` | iot-kafka | Шина сообщений | +| 2 | `nubes_redis` | iot-redis | Кэш дашборда | +| 3 | `nubes_mongodb` | iot-mongo | Постоянное хранение | +| 4 | `nubes_flask` | iot-producer | Генератор IoT → Kafka | +| 5 | `nubes_flask` | iot-consumer | Kafka → Redis + MongoDB | +| 6 | `nubes_nodejs` | iot-dashboard | Redis → графики Chart.js | + +## Git-репозитории + +| Репо | Назначение | +|------|-----------| +| `Nail/tf-iot-producer` | Flask — IoT продюсер (Kafka TLS) | +| `Nail/tf-iot-consumer` | Flask — Kafka → Redis + MongoDB | +| `Nail/tf-iot-dashboard` | Node.js — дашборд Chart.js | + +## Поток данных + +``` +Producer (Flask) → Kafka → Consumer (Flask) → Redis (кэш) + → MongoDB (история) +Dashboard (Node.js) ← Redis (API /api/latest, /api/counters, /api/recent) +``` + +## Реальные ключи (определены опытным путём) + +### Kafka +- bootstrap: `state_out_flat["internalConnect.bootstrap.fqdn"]:9093` — **TLS** +- vault: `vault_secrets["users"]["iot-app"]["user.password"]` + +### Redis +- host: `state_out_flat["internalConnect.master"]` +- vault: `vault_secrets["users"]["default"]["password"]` + +### MongoDB +- host: `state_out_flat["internalConnect"]` +- port: 27017 +- vault: `vault_secrets["users"]["admin"]["password"]` + +## Структура приложений + +- **Flask**: `site/app.py` + `requirements.txt` (БЕЗ Procfile) +- **Node.js**: `server.js` + `package.json` (`npm start`) + +## Неудачные сервисы + +| Сервис | Причина | +|--------|--------| +| `nubes_clickhouse` | GitHub release assets timeout | +| `nubes_superset` | IAM 408 timeout | +| `nubes_akhq` | `toBoolean() on null` — баг платформы |