From 336e6e3795b1bf6ab7d1a7a2537f565765b49928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Tue, 21 Jul 2026 17:33:03 +0400 Subject: [PATCH] =?UTF-8?q?feat:=20IoT=20Kafka=20demo=20=E2=80=94=20Terraf?= =?UTF-8?q?orm=20+=20history=20(test=20stand=20fail,=20moving=20to=20dev)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TEST_STAND/IOT_KAFKA_DEMO/clickhouse.tf | 66 +++++++++++++ TEST_STAND/IOT_KAFKA_DEMO/consumer.tf | 53 +++++++++++ TEST_STAND/IOT_KAFKA_DEMO/kafka.tf | 45 +++++++++ TEST_STAND/IOT_KAFKA_DEMO/locals.tf | 93 +++++++++++++++++++ TEST_STAND/IOT_KAFKA_DEMO/main.tf | 29 ++++++ TEST_STAND/IOT_KAFKA_DEMO/producer.tf | 46 +++++++++ TEST_STAND/IOT_KAFKA_DEMO/superset.tf | 14 +++ .../IOT_KAFKA_DEMO/terraform.tfvars.example | 4 + apps/iot-demo/HISTORY/01-init.md | 34 +++++++ apps/iot-demo/HISTORY/02-repos.md | 29 ++++++ apps/iot-demo/HISTORY/03-terraform.md | 45 +++++++++ apps/iot-demo/HISTORY/04-plan-success.md | 41 ++++++++ apps/iot-demo/HISTORY/05-test-stand-fail.md | 37 ++++++++ 13 files changed, 536 insertions(+) create mode 100644 TEST_STAND/IOT_KAFKA_DEMO/clickhouse.tf create mode 100644 TEST_STAND/IOT_KAFKA_DEMO/consumer.tf create mode 100644 TEST_STAND/IOT_KAFKA_DEMO/kafka.tf create mode 100644 TEST_STAND/IOT_KAFKA_DEMO/locals.tf create mode 100644 TEST_STAND/IOT_KAFKA_DEMO/main.tf create mode 100644 TEST_STAND/IOT_KAFKA_DEMO/producer.tf create mode 100644 TEST_STAND/IOT_KAFKA_DEMO/superset.tf create mode 100644 TEST_STAND/IOT_KAFKA_DEMO/terraform.tfvars.example create mode 100644 apps/iot-demo/HISTORY/01-init.md create mode 100644 apps/iot-demo/HISTORY/02-repos.md create mode 100644 apps/iot-demo/HISTORY/03-terraform.md create mode 100644 apps/iot-demo/HISTORY/04-plan-success.md create mode 100644 apps/iot-demo/HISTORY/05-test-stand-fail.md diff --git a/TEST_STAND/IOT_KAFKA_DEMO/clickhouse.tf b/TEST_STAND/IOT_KAFKA_DEMO/clickhouse.tf new file mode 100644 index 0000000..720b6df --- /dev/null +++ b/TEST_STAND/IOT_KAFKA_DEMO/clickhouse.tf @@ -0,0 +1,66 @@ +# ============================================================================= +# ClickHouse — аналитическая БД +# ============================================================================= +resource "nubes_clickhouse" "main" { + resource_name = local.ch_resource_name + + startup_configuration = { + resource_realm = var.realm + } + + cluster_configuration = { + cpu = local.ch_cpu + memory = local.ch_memory + disk = local.ch_disk + replicas = local.ch_replicas + shards = local.ch_shards + } + + access_configuration = { + master_ip_space = "no-needed" + master_access_list = jsonencode(["10.0.0.0/8"]) + } + + clickhouse_configuration = { + version = local.ch_version + } + + clickhouse_keeper_configuration = { + cpu = 500 + memory = 512 + replicas = 1 + } + + backup_configuration = { + s3_uid = var.s3_name + retain = 7 + schedule = "0 0 * * *" + } + + autoscale_configuration = { + enabled = false + percent = 10 + quota = 100 + schedule = 0 + } + + operation_timeout = local.ch_timeout +} + +# ----------------------------------------------------------------------------- +# ClickHouse Database +# ----------------------------------------------------------------------------- +resource "nubes_clickhouse_database" "iot" { + clickhouse_id = nubes_clickhouse.main.id + db_name = local.ch_db_name +} + +# ----------------------------------------------------------------------------- +# ClickHouse User +# ----------------------------------------------------------------------------- +resource "nubes_clickhouse_user" "writer" { + clickhouse_id = nubes_clickhouse.main.id + username = local.ch_username + db_name = nubes_clickhouse_database.iot.db_name + access_hosts = "0.0.0.0/0" +} diff --git a/TEST_STAND/IOT_KAFKA_DEMO/consumer.tf b/TEST_STAND/IOT_KAFKA_DEMO/consumer.tf new file mode 100644 index 0000000..2fa845b --- /dev/null +++ b/TEST_STAND/IOT_KAFKA_DEMO/consumer.tf @@ -0,0 +1,53 @@ +# ============================================================================= +# Flask Consumer — читает Kafka, пишет в ClickHouse +# ============================================================================= +locals { + consumer_kafka_secrets = try(jsondecode(nubes_kafka.main.vault_secrets["users"]), jsondecode(nubes_kafka.main.vault_secrets["secrets"]), tomap({})) + consumer_ch_secrets = try(jsondecode(nubes_clickhouse.main.vault_secrets["users"]), jsondecode(nubes_clickhouse.main.vault_secrets["secrets"]), tomap({})) + consumer_kafka_brokers = try(nubes_kafka.main.state_out_flat["bootstrapServers"], "") + consumer_kafka_password = try(nonsensitive(lookup(local.consumer_kafka_secrets, "iot-app", tomap({})).password), "") + consumer_ch_host = try(nubes_clickhouse.main.state_out_flat["internalMaster"], "") + consumer_ch_password = try(nonsensitive(lookup(local.consumer_ch_secrets, "iot_writer", tomap({})).password), "") +} + +resource "nubes_flask" "consumer" { + resource_name = local.consumer_resource_name + + startup_configuration = { + resource_realm = var.realm + } + + cluster_configuration = { + cpu = local.consumer_cpu + memory = local.consumer_memory + replicas = local.consumer_replicas + } + + access_configuration = { + domain = local.consumer_domain + } + + app_configuration = { + version = "3.12" + git_path = local.consumer_git_path + health_path = "/health" + } + + git_revision = local.consumer_git_revision + operation_timeout = local.consumer_timeout + + json_env = jsonencode({ + KAFKA_BROKERS = local.consumer_kafka_brokers + KAFKA_TOPIC = local.kafka_topic_name + KAFKA_USERNAME = local.kafka_username + KAFKA_PASSWORD = local.consumer_kafka_password + KAFKA_GROUP_ID = "iot-consumer-group" + CH_HOST = local.consumer_ch_host + CH_PORT = local.ch_http_port + CH_USER = local.ch_username + CH_PASSWORD = local.consumer_ch_password + CH_DATABASE = local.ch_db_name + }) + + depends_on = [nubes_kafka.main, nubes_clickhouse.main] +} diff --git a/TEST_STAND/IOT_KAFKA_DEMO/kafka.tf b/TEST_STAND/IOT_KAFKA_DEMO/kafka.tf new file mode 100644 index 0000000..99f65a4 --- /dev/null +++ b/TEST_STAND/IOT_KAFKA_DEMO/kafka.tf @@ -0,0 +1,45 @@ +# ============================================================================= +# Kafka — шина сообщений +# ============================================================================= +resource "nubes_kafka" "main" { + resource_name = local.kafka_resource_name + + startup_configuration = { + resource_realm = var.realm + } + + cluster_configuration = { + cpu = local.kafka_cpu + memory = local.kafka_memory + disk = local.kafka_disk + replicas = local.kafka_replicas + } + + access_configuration = { + master_access_list = jsonencode(["10.0.0.0/8"]) + } + + operation_timeout = local.kafka_timeout +} + +# ----------------------------------------------------------------------------- +# Kafka Topic — iot-events +# ----------------------------------------------------------------------------- +resource "nubes_kafka_topic" "events" { + kafka_id = nubes_kafka.main.id + name_topic = local.kafka_topic_name + partitions = local.kafka_topic_parts + replicas = local.kafka_topic_repl +} + +# ----------------------------------------------------------------------------- +# Kafka User — общий для producer и consumer +# ----------------------------------------------------------------------------- +resource "nubes_kafka_user" "app" { + kafka_id = nubes_kafka.main.id + username = local.kafka_username + name_topic = nubes_kafka_topic.events.name_topic + operations = "Create,Describe,Read,Write" + access_hosts = "*" + group = "*" +} diff --git a/TEST_STAND/IOT_KAFKA_DEMO/locals.tf b/TEST_STAND/IOT_KAFKA_DEMO/locals.tf new file mode 100644 index 0000000..24043f2 --- /dev/null +++ b/TEST_STAND/IOT_KAFKA_DEMO/locals.tf @@ -0,0 +1,93 @@ +# ============================================================================= +# locals.tf — все настраиваемые значения IOT_KAFKA_DEMO +# Kafka + ClickHouse + Superset + Flask (producer + consumer) +# ============================================================================= + +locals { + + # ═══════════════════════════════════════════════════════════════════════════ + # Kafka + # ═══════════════════════════════════════════════════════════════════════════ + + kafka_resource_name = "iot-kafka" + kafka_cpu = 500 + kafka_memory = 1024 + kafka_disk = 10 + kafka_replicas = 1 + kafka_version = "3.7" + kafka_timeout = "15m" + + # Kafka — topic + kafka_topic_name = "iot-events" + kafka_topic_parts = 3 + kafka_topic_repl = 1 + + # Kafka — user (общий для producer и consumer) + kafka_username = "iot-app" + + # ═══════════════════════════════════════════════════════════════════════════ + # ClickHouse + # ═══════════════════════════════════════════════════════════════════════════ + + ch_resource_name = "iot-ch" + ch_cpu = 500 + ch_memory = 1024 + ch_disk = 10 + ch_replicas = 1 + ch_shards = 1 + ch_version = "24.8" + ch_timeout = "15m" + + # ClickHouse — database + ch_db_name = "iot" + + # ClickHouse — user + ch_username = "iot_writer" + ch_role = "ddl_user" + + # ═══════════════════════════════════════════════════════════════════════════ + # Superset + # ═══════════════════════════════════════════════════════════════════════════ + + superset_resource_name = "iot-superset" + superset_domain = "iotsuperset" + superset_cpu = 500 + superset_memory = 1024 + superset_disk = 5 + superset_instances = 1 + superset_email = "admin" + superset_timeout = "15m" + + # ═══════════════════════════════════════════════════════════════════════════ + # Flask — Producer + # ═══════════════════════════════════════════════════════════════════════════ + + producer_resource_name = "iot-producer" + producer_domain = "iotproducer" + producer_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-producer.git" + producer_git_revision = "master" + producer_cpu = 300 + producer_memory = 256 + producer_replicas = 1 + producer_timeout = "11m" + + # ═══════════════════════════════════════════════════════════════════════════ + # Flask — Consumer + # ═══════════════════════════════════════════════════════════════════════════ + + consumer_resource_name = "iot-consumer" + consumer_domain = "iotconsumer" + consumer_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-consumer.git" + consumer_git_revision = "master" + consumer_cpu = 300 + consumer_memory = 256 + consumer_replicas = 1 + consumer_timeout = "11m" + + # ═══════════════════════════════════════════════════════════════════════════ + # Общие + # ═══════════════════════════════════════════════════════════════════════════ + + kafka_port = "9092" + ch_http_port = "8123" +} diff --git a/TEST_STAND/IOT_KAFKA_DEMO/main.tf b/TEST_STAND/IOT_KAFKA_DEMO/main.tf new file mode 100644 index 0000000..149bb4f --- /dev/null +++ b/TEST_STAND/IOT_KAFKA_DEMO/main.tf @@ -0,0 +1,29 @@ +terraform { + required_providers { + nubes = { + source = "registry.kube5s.ru/nubes-test/nubes" + version = "5.1.16" + } + } +} + +variable "api_token" { + type = string + sensitive = true + description = "Nubes API token" +} + +variable "realm" { + type = string + description = "resource_realm parameter for all resources" +} + +variable "s3_name" { + type = string + description = "S3 user name for backups" +} + +provider "nubes" { + api_token = var.api_token + api_endpoint = "https://lk-api-gateway-test.ngcloud.ru/api/v1/svc" +} diff --git a/TEST_STAND/IOT_KAFKA_DEMO/producer.tf b/TEST_STAND/IOT_KAFKA_DEMO/producer.tf new file mode 100644 index 0000000..9be87c3 --- /dev/null +++ b/TEST_STAND/IOT_KAFKA_DEMO/producer.tf @@ -0,0 +1,46 @@ +# ============================================================================= +# Flask Producer — генерирует IoT-события, шлёт в Kafka +# ============================================================================= +locals { + # vault_secrets для доступа к паролям (ключ зависит от сервиса: users/secrets) + producer_kafka_secrets = try(jsondecode(nubes_kafka.main.vault_secrets["users"]), jsondecode(nubes_kafka.main.vault_secrets["secrets"]), tomap({})) + producer_kafka_brokers = try(nubes_kafka.main.state_out_flat["bootstrapServers"], "") + producer_kafka_password = try(nonsensitive(lookup(local.producer_kafka_secrets, "iot-app", tomap({})).password), "") +} + +resource "nubes_flask" "producer" { + resource_name = local.producer_resource_name + + startup_configuration = { + resource_realm = var.realm + } + + cluster_configuration = { + cpu = local.producer_cpu + memory = local.producer_memory + replicas = local.producer_replicas + } + + access_configuration = { + domain = local.producer_domain + } + + app_configuration = { + version = "3.12" + git_path = local.producer_git_path + health_path = "/health" + } + + git_revision = local.producer_git_revision + operation_timeout = local.producer_timeout + + json_env = jsonencode({ + KAFKA_BROKERS = local.producer_kafka_brokers + KAFKA_TOPIC = local.kafka_topic_name + KAFKA_USERNAME = local.kafka_username + KAFKA_PASSWORD = local.producer_kafka_password + PRODUCE_INTERVAL = "3" + }) + + depends_on = [nubes_kafka.main] +} diff --git a/TEST_STAND/IOT_KAFKA_DEMO/superset.tf b/TEST_STAND/IOT_KAFKA_DEMO/superset.tf new file mode 100644 index 0000000..31d926f --- /dev/null +++ b/TEST_STAND/IOT_KAFKA_DEMO/superset.tf @@ -0,0 +1,14 @@ +# ============================================================================= +# Superset — дашборды +# ============================================================================= +resource "nubes_superset" "dash" { + resource_name = local.superset_resource_name + resource_realm = var.realm + domain = local.superset_domain + resource_c_p_u = local.superset_cpu + resource_memory = local.superset_memory + resource_disk = local.superset_disk + resource_instances = local.superset_instances + emails = local.superset_email + operation_timeout = local.superset_timeout +} diff --git a/TEST_STAND/IOT_KAFKA_DEMO/terraform.tfvars.example b/TEST_STAND/IOT_KAFKA_DEMO/terraform.tfvars.example new file mode 100644 index 0000000..c41989a --- /dev/null +++ b/TEST_STAND/IOT_KAFKA_DEMO/terraform.tfvars.example @@ -0,0 +1,4 @@ +# Скопировать в terraform.tfvars и заполнить +api_token = "ВАШ_API_ТОКЕН" +realm = "ВАШ_REALM" +s3_name = "ВАШ_S3_USER" diff --git a/apps/iot-demo/HISTORY/01-init.md b/apps/iot-demo/HISTORY/01-init.md new file mode 100644 index 0000000..d806b5b --- /dev/null +++ b/apps/iot-demo/HISTORY/01-init.md @@ -0,0 +1,34 @@ +# Шаг 1 — Инициализация проекта + +**Дата:** 2026-07-21 + +## Создана структура папок + +``` +TEST_STAND/IOT_KAFKA_DEMO/ # Terraform-конфиги +TEST_STAND/IOT_KAFKA_DEMO/HISTORY/ # Пошаговая история +apps/iot-producer/ # Flask — продюсер событий в Kafka +apps/iot-consumer/ # Flask — consumer (Kafka → ClickHouse) +``` + +## Архитектура демо + +``` +Flask (producer) → Kafka → Flask (consumer) → ClickHouse → Superset (dashboard) +``` + +## Сервисы облака Nubes + +| Ресурс | Роль | +|--------|------| +| `nubes_flask` (producer) | Генерирует IoT-события, шлёт в Kafka | +| `nubes_kafka` | Шина сообщений | +| `nubes_flask` (consumer) | Читает Kafka, пишет в ClickHouse | +| `nubes_clickhouse` | Аналитическая time-series БД | +| `nubes_superset` | Дашборд (температура, влажность, энергия) | + +## Данные + +IoT-симуляция «умный дом»: +- `sensor_id`, `device_type`, `location`, `value`, `unit`, `timestamp` +- Генерация каждые 2-5 секунд diff --git a/apps/iot-demo/HISTORY/02-repos.md b/apps/iot-demo/HISTORY/02-repos.md new file mode 100644 index 0000000..ed45982 --- /dev/null +++ b/apps/iot-demo/HISTORY/02-repos.md @@ -0,0 +1,29 @@ +# Шаг 2 — Репозитории и код приложений + +**Дата:** 2026-07-21 + +## Созданы Git-репозитории на Gitea + +| Репозиторий | URL | +|-------------|-----| +| tf-iot-producer | https://gitea.services.ngcloud.ru/Nail/tf-iot-producer | +| tf-iot-consumer | https://gitea.services.ngcloud.ru/Nail/tf-iot-consumer | + +## Producer (tf-iot-producer) + +`app.py` — Flask + KafkaProducer: +- Генерирует фейковые IoT-события (8 датчиков: температура, влажность, энергия) +- Шлёт в Kafka каждые `PRODUCE_INTERVAL` секунд (по умолчанию 3) +- Топик и брокеры — из env (задаёт Terraform) +- SASL_PLAINTEXT аутентификация +- `GET /` — статистика (events_sent, errors) +- `GET /health` — healthcheck + +## Consumer (tf-iot-consumer) + +`app.py` — Flask + KafkaConsumer + ClickHouse: +- Читает события из Kafka, пишет батчами по 10 в ClickHouse +- Создаёт таблицу `iot_events` (MergeTree) при старте +- SASL_PLAINTEXT → Kafka, HTTP → ClickHouse +- `GET /` — статистика (events_consumed, events_in_db) +- `GET /health` — healthcheck diff --git a/apps/iot-demo/HISTORY/03-terraform.md b/apps/iot-demo/HISTORY/03-terraform.md new file mode 100644 index 0000000..f2edb6a --- /dev/null +++ b/apps/iot-demo/HISTORY/03-terraform.md @@ -0,0 +1,45 @@ +# Шаг 3 — Terraform-конфиги + +**Дата:** 2026-07-21 + +## Созданы файлы + +| Файл | Назначение | +|------|-----------| +| `main.tf` | Provider, variables | +| `locals.tf` | Все настраиваемые параметры | +| `kafka.tf` | `nubes_kafka` + topic + user | +| `clickhouse.tf` | `nubes_clickhouse` + database + user | +| `superset.tf` | `nubes_superset` | +| `producer.tf` | `nubes_flask` — IoT продюсер | +| `consumer.tf` | `nubes_flask` — IoT consumer | +| `terraform.tfvars.example` | Пример переменных | + +## Ресурсы (5 штук) + +``` +nubes_kafka.main + ├── nubes_kafka_topic.events (iot-events) + └── nubes_kafka_user.app (iot-app, read+write) + +nubes_clickhouse.main + ├── nubes_clickhouse_database.iot (iot) + └── nubes_clickhouse_user.writer (iot_writer) + +nubes_superset.dash + +nubes_flask.producer → Kafka +nubes_flask.consumer → Kafka → ClickHouse +``` + +## Передача секретов + +- Kafka: `vault_secrets["users"]` → пароль пользователя +- ClickHouse: `vault_secrets["users"]` → пароль пользователя +- Bootstrap servers: `state_out_flat["bootstrapServers"]` +- ClickHouse host: `state_out_flat["internalMaster"]` + +## ⚠️ Требует проверки + +- Точные ключи `state_out_flat` для Kafka (`bootstrapServers`) и ClickHouse (`internalMaster`) — нужна верификация на реальном стенде +- Формат `vault_secrets["users"]` для Kafka и ClickHouse — аналогичен PostgreSQL, но требует проверки diff --git a/apps/iot-demo/HISTORY/04-plan-success.md b/apps/iot-demo/HISTORY/04-plan-success.md new file mode 100644 index 0000000..a034250 --- /dev/null +++ b/apps/iot-demo/HISTORY/04-plan-success.md @@ -0,0 +1,41 @@ +# Шаг 4 — Terraform plan (успех) + +**Дата:** 2026-07-21 + +## Результат + +`terraform plan` — **успешно**. 9 ресурсов, 0 ошибок. + +## Исправления + +1. **Синтаксис vault_secrets** — `${local.var}` нельзя использовать после точки. + Решение: декодировать в отдельный local, хардкодить username (как в CRUD): + ```hcl + locals { + kafka_secrets = jsondecode(nubes_kafka.main.vault_secrets["users"]) + password = nonsensitive(local.kafka_secrets["iot-app"].password) + } + ``` + +2. **ClickHouse обязательные блоки** — добавлены: + - `clickhouse_keeper_configuration` (cpu/memory/replicas) + - `backup_configuration` (s3_uid/retain/schedule) + - `autoscale_configuration` (enabled/percent/quota/schedule) + +## План (9 ресурсов) + +| # | Ресурс | Имя | +|---|--------|-----| +| 1 | nubes_kafka | iot-kafka | +| 2 | nubes_kafka_topic | iot-events | +| 3 | nubes_kafka_user | iot-app | +| 4 | nubes_clickhouse | iot-clickhouse | +| 5 | nubes_clickhouse_database | iot | +| 6 | nubes_clickhouse_user | iot_writer | +| 7 | nubes_flask | iot-producer | +| 8 | nubes_flask | iot-consumer | +| 9 | nubes_superset | iot-superset | + +## ⚠️ Остаётся неизвестным + +Ключи `state_out_flat` (`bootstrapServers`, `internalMaster`) и формат `vault_secrets` — станут известны только после `apply`. Завёрнуты в `try()` чтобы не упасть, но реальные значения будут пустыми до создания инстансов. diff --git a/apps/iot-demo/HISTORY/05-test-stand-fail.md b/apps/iot-demo/HISTORY/05-test-stand-fail.md new file mode 100644 index 0000000..43a55c8 --- /dev/null +++ b/apps/iot-demo/HISTORY/05-test-stand-fail.md @@ -0,0 +1,37 @@ +# Шаг 5 — Apply на тестовом стенде (неудача) + +**Дата:** 2026-07-21 + +## Результат + +Тестовый стенд (`k8s-3` и `k8s-4`) неработоспособен — инфраструктурные проблемы. + +## Что создалось + +| Ресурс | Статус | +|--------|--------| +| nubes_kafka.main (iot-kafka) | ✅ Создан | +| nubes_kafka_topic.events | ✅ Создан | +| nubes_kafka_user.app | ❌ operations формат | +| nubes_clickhouse.main (iot-ch) | ❌ helm.altinity.com timeout | +| nubes_superset.dash | ❌ IAM 408 timeout | +| nubes_flask.producer | ❌ pod не запустился | +| nubes_flask.consumer | ❌ (зависит от CH) | + +## Ошибки + +1. **ClickHouse**: GitHub/Helm repo недоступен — `context deadline exceeded` +2. **Superset**: IAM error 408 — auth-api-test таймаутит +3. **Flask producer**: поды не стартуют — `jlib.k8s [correctReplicaActive] ERROR` +4. **Kafka user**: `operations` должен быть `Create,Describe,Read,Write` (исправлено) + +## Исправления в коде + +- `producer.tf` / `consumer.tf`: `vault_secrets` обёрнуты в `try()` с fallback на `"secrets"` +- `kafka.tf`: `operations = "Create,Describe,Read,Write"` (было `"read,write"`) +- `locals.tf`: `ch_resource_name = "iot-ch"` (было `"iot-clickhouse"`) +- `terraform.tfvars`: `realm = "k8s-4-sandbox-nubes-ru"` + +## Решение + +Перенос на DEV-стенд.