feat: IoT Kafka demo — Terraform + history (test stand fail, moving to dev)

TEST_STAND/IOT_KAFKA_DEMO/clickhouse.tf

@@ -0,0 +1,66 @@
+# =============================================================================
+# ClickHouse — аналитическая БД
+# =============================================================================
+resource "nubes_clickhouse" "main" {
+  resource_name = local.ch_resource_name
+
+  startup_configuration = {
+    resource_realm = var.realm
+  }
+
+  cluster_configuration = {
+    cpu      = local.ch_cpu
+    memory   = local.ch_memory
+    disk     = local.ch_disk
+    replicas = local.ch_replicas
+    shards   = local.ch_shards
+  }
+
+  access_configuration = {
+    master_ip_space    = "no-needed"
+    master_access_list = jsonencode(["10.0.0.0/8"])
+  }
+
+  clickhouse_configuration = {
+    version = local.ch_version
+  }
+
+  clickhouse_keeper_configuration = {
+    cpu      = 500
+    memory   = 512
+    replicas = 1
+  }
+
+  backup_configuration = {
+    s3_uid   = var.s3_name
+    retain   = 7
+    schedule = "0 0 * * *"
+  }
+
+  autoscale_configuration = {
+    enabled  = false
+    percent  = 10
+    quota    = 100
+    schedule = 0
+  }
+
+  operation_timeout = local.ch_timeout
+}
+
+# -----------------------------------------------------------------------------
+# ClickHouse Database
+# -----------------------------------------------------------------------------
+resource "nubes_clickhouse_database" "iot" {
+  clickhouse_id = nubes_clickhouse.main.id
+  db_name       = local.ch_db_name
+}
+
+# -----------------------------------------------------------------------------
+# ClickHouse User
+# -----------------------------------------------------------------------------
+resource "nubes_clickhouse_user" "writer" {
+  clickhouse_id = nubes_clickhouse.main.id
+  username      = local.ch_username
+  db_name       = nubes_clickhouse_database.iot.db_name
+  access_hosts  = "0.0.0.0/0"
+}

TEST_STAND/IOT_KAFKA_DEMO/consumer.tf

@@ -0,0 +1,53 @@
+# =============================================================================
+# Flask Consumer — читает Kafka, пишет в ClickHouse
+# =============================================================================
+locals {
+  consumer_kafka_secrets = try(jsondecode(nubes_kafka.main.vault_secrets["users"]), jsondecode(nubes_kafka.main.vault_secrets["secrets"]), tomap({}))
+  consumer_ch_secrets    = try(jsondecode(nubes_clickhouse.main.vault_secrets["users"]), jsondecode(nubes_clickhouse.main.vault_secrets["secrets"]), tomap({}))
+  consumer_kafka_brokers = try(nubes_kafka.main.state_out_flat["bootstrapServers"], "")
+  consumer_kafka_password = try(nonsensitive(lookup(local.consumer_kafka_secrets, "iot-app", tomap({})).password), "")
+  consumer_ch_host     = try(nubes_clickhouse.main.state_out_flat["internalMaster"], "")
+  consumer_ch_password = try(nonsensitive(lookup(local.consumer_ch_secrets, "iot_writer", tomap({})).password), "")
+}
+
+resource "nubes_flask" "consumer" {
+  resource_name = local.consumer_resource_name
+
+  startup_configuration = {
+    resource_realm = var.realm
+  }
+
+  cluster_configuration = {
+    cpu      = local.consumer_cpu
+    memory   = local.consumer_memory
+    replicas = local.consumer_replicas
+  }
+
+  access_configuration = {
+    domain = local.consumer_domain
+  }
+
+  app_configuration = {
+    version     = "3.12"
+    git_path    = local.consumer_git_path
+    health_path = "/health"
+  }
+
+  git_revision      = local.consumer_git_revision
+  operation_timeout = local.consumer_timeout
+
+  json_env = jsonencode({
+    KAFKA_BROKERS  = local.consumer_kafka_brokers
+    KAFKA_TOPIC    = local.kafka_topic_name
+    KAFKA_USERNAME = local.kafka_username
+    KAFKA_PASSWORD = local.consumer_kafka_password
+    KAFKA_GROUP_ID = "iot-consumer-group"
+    CH_HOST        = local.consumer_ch_host
+    CH_PORT        = local.ch_http_port
+    CH_USER        = local.ch_username
+    CH_PASSWORD    = local.consumer_ch_password
+    CH_DATABASE    = local.ch_db_name
+  })
+
+  depends_on = [nubes_kafka.main, nubes_clickhouse.main]
+}

TEST_STAND/IOT_KAFKA_DEMO/kafka.tf

@@ -0,0 +1,45 @@
+# =============================================================================
+# Kafka — шина сообщений
+# =============================================================================
+resource "nubes_kafka" "main" {
+  resource_name = local.kafka_resource_name
+
+  startup_configuration = {
+    resource_realm = var.realm
+  }
+
+  cluster_configuration = {
+    cpu      = local.kafka_cpu
+    memory   = local.kafka_memory
+    disk     = local.kafka_disk
+    replicas = local.kafka_replicas
+  }
+
+  access_configuration = {
+    master_access_list = jsonencode(["10.0.0.0/8"])
+  }
+
+  operation_timeout = local.kafka_timeout
+}
+
+# -----------------------------------------------------------------------------
+# Kafka Topic — iot-events
+# -----------------------------------------------------------------------------
+resource "nubes_kafka_topic" "events" {
+  kafka_id   = nubes_kafka.main.id
+  name_topic = local.kafka_topic_name
+  partitions = local.kafka_topic_parts
+  replicas   = local.kafka_topic_repl
+}
+
+# -----------------------------------------------------------------------------
+# Kafka User — общий для producer и consumer
+# -----------------------------------------------------------------------------
+resource "nubes_kafka_user" "app" {
+  kafka_id     = nubes_kafka.main.id
+  username     = local.kafka_username
+  name_topic   = nubes_kafka_topic.events.name_topic
+  operations   = "Create,Describe,Read,Write"
+  access_hosts = "*"
+  group        = "*"
+}

TEST_STAND/IOT_KAFKA_DEMO/locals.tf

@@ -0,0 +1,93 @@
+# =============================================================================
+# locals.tf — все настраиваемые значения IOT_KAFKA_DEMO
+# Kafka + ClickHouse + Superset + Flask (producer + consumer)
+# =============================================================================
+
+locals {
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Kafka
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  kafka_resource_name = "iot-kafka"
+  kafka_cpu           = 500
+  kafka_memory        = 1024
+  kafka_disk          = 10
+  kafka_replicas      = 1
+  kafka_version       = "3.7"
+  kafka_timeout       = "15m"
+
+  # Kafka — topic
+  kafka_topic_name  = "iot-events"
+  kafka_topic_parts = 3
+  kafka_topic_repl  = 1
+
+  # Kafka — user (общий для producer и consumer)
+  kafka_username = "iot-app"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # ClickHouse
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  ch_resource_name = "iot-ch"
+  ch_cpu           = 500
+  ch_memory        = 1024
+  ch_disk          = 10
+  ch_replicas      = 1
+  ch_shards        = 1
+  ch_version       = "24.8"
+  ch_timeout       = "15m"
+
+  # ClickHouse — database
+  ch_db_name = "iot"
+
+  # ClickHouse — user
+  ch_username = "iot_writer"
+  ch_role     = "ddl_user"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Superset
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  superset_resource_name = "iot-superset"
+  superset_domain        = "iotsuperset"
+  superset_cpu           = 500
+  superset_memory        = 1024
+  superset_disk          = 5
+  superset_instances     = 1
+  superset_email         = "admin"
+  superset_timeout       = "15m"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Flask — Producer
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  producer_resource_name = "iot-producer"
+  producer_domain        = "iotproducer"
+  producer_git_path      = "https://gitea.services.ngcloud.ru/Nail/tf-iot-producer.git"
+  producer_git_revision  = "master"
+  producer_cpu           = 300
+  producer_memory        = 256
+  producer_replicas      = 1
+  producer_timeout       = "11m"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Flask — Consumer
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  consumer_resource_name = "iot-consumer"
+  consumer_domain        = "iotconsumer"
+  consumer_git_path      = "https://gitea.services.ngcloud.ru/Nail/tf-iot-consumer.git"
+  consumer_git_revision  = "master"
+  consumer_cpu           = 300
+  consumer_memory        = 256
+  consumer_replicas      = 1
+  consumer_timeout       = "11m"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Общие
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  kafka_port   = "9092"
+  ch_http_port = "8123"
+}

TEST_STAND/IOT_KAFKA_DEMO/main.tf

@@ -0,0 +1,29 @@
+terraform {
+  required_providers {
+    nubes = {
+      source  = "registry.kube5s.ru/nubes-test/nubes"
+      version = "5.1.16"
+    }
+  }
+}
+
+variable "api_token" {
+  type        = string
+  sensitive   = true
+  description = "Nubes API token"
+}
+
+variable "realm" {
+  type        = string
+  description = "resource_realm parameter for all resources"
+}
+
+variable "s3_name" {
+  type        = string
+  description = "S3 user name for backups"
+}
+
+provider "nubes" {
+  api_token    = var.api_token
+  api_endpoint = "https://lk-api-gateway-test.ngcloud.ru/api/v1/svc"
+}

TEST_STAND/IOT_KAFKA_DEMO/producer.tf

@@ -0,0 +1,46 @@
+# =============================================================================
+# Flask Producer — генерирует IoT-события, шлёт в Kafka
+# =============================================================================
+locals {
+  # vault_secrets для доступа к паролям (ключ зависит от сервиса: users/secrets)
+  producer_kafka_secrets = try(jsondecode(nubes_kafka.main.vault_secrets["users"]), jsondecode(nubes_kafka.main.vault_secrets["secrets"]), tomap({}))
+  producer_kafka_brokers = try(nubes_kafka.main.state_out_flat["bootstrapServers"], "")
+  producer_kafka_password = try(nonsensitive(lookup(local.producer_kafka_secrets, "iot-app", tomap({})).password), "")
+}
+
+resource "nubes_flask" "producer" {
+  resource_name = local.producer_resource_name
+
+  startup_configuration = {
+    resource_realm = var.realm
+  }
+
+  cluster_configuration = {
+    cpu      = local.producer_cpu
+    memory   = local.producer_memory
+    replicas = local.producer_replicas
+  }
+
+  access_configuration = {
+    domain = local.producer_domain
+  }
+
+  app_configuration = {
+    version     = "3.12"
+    git_path    = local.producer_git_path
+    health_path = "/health"
+  }
+
+  git_revision      = local.producer_git_revision
+  operation_timeout = local.producer_timeout
+
+  json_env = jsonencode({
+    KAFKA_BROKERS   = local.producer_kafka_brokers
+    KAFKA_TOPIC     = local.kafka_topic_name
+    KAFKA_USERNAME  = local.kafka_username
+    KAFKA_PASSWORD  = local.producer_kafka_password
+    PRODUCE_INTERVAL = "3"
+  })
+
+  depends_on = [nubes_kafka.main]
+}

TEST_STAND/IOT_KAFKA_DEMO/superset.tf

@@ -0,0 +1,14 @@
+# =============================================================================
+# Superset — дашборды
+# =============================================================================
+resource "nubes_superset" "dash" {
+  resource_name     = local.superset_resource_name
+  resource_realm    = var.realm
+  domain            = local.superset_domain
+  resource_c_p_u    = local.superset_cpu
+  resource_memory   = local.superset_memory
+  resource_disk     = local.superset_disk
+  resource_instances = local.superset_instances
+  emails            = local.superset_email
+  operation_timeout = local.superset_timeout
+}

TEST_STAND/IOT_KAFKA_DEMO/terraform.tfvars.example

@@ -0,0 +1,4 @@
+# Скопировать в terraform.tfvars и заполнить
+api_token = "ВАШ_API_ТОКЕН"
+realm     = "ВАШ_REALM"
+s3_name   = "ВАШ_S3_USER"

apps/iot-demo/HISTORY/01-init.md

@@ -0,0 +1,34 @@
+# Шаг 1 — Инициализация проекта
+
+**Дата:** 2026-07-21
+
+## Создана структура папок
+
+```
+TEST_STAND/IOT_KAFKA_DEMO/       # Terraform-конфиги
+TEST_STAND/IOT_KAFKA_DEMO/HISTORY/  # Пошаговая история
+apps/iot-producer/               # Flask — продюсер событий в Kafka
+apps/iot-consumer/               # Flask — consumer (Kafka → ClickHouse)
+```
+
+## Архитектура демо
+
+```
+Flask (producer) → Kafka → Flask (consumer) → ClickHouse → Superset (dashboard)
+```
+
+## Сервисы облака Nubes
+
+| Ресурс | Роль |
+|--------|------|
+| `nubes_flask` (producer) | Генерирует IoT-события, шлёт в Kafka |
+| `nubes_kafka` | Шина сообщений |
+| `nubes_flask` (consumer) | Читает Kafka, пишет в ClickHouse |
+| `nubes_clickhouse` | Аналитическая time-series БД |
+| `nubes_superset` | Дашборд (температура, влажность, энергия) |
+
+## Данные
+
+IoT-симуляция «умный дом»:
+- `sensor_id`, `device_type`, `location`, `value`, `unit`, `timestamp`
+- Генерация каждые 2-5 секунд

apps/iot-demo/HISTORY/02-repos.md

@@ -0,0 +1,29 @@
+# Шаг 2 — Репозитории и код приложений
+
+**Дата:** 2026-07-21
+
+## Созданы Git-репозитории на Gitea
+
+| Репозиторий | URL |
+|-------------|-----|
+| tf-iot-producer | https://gitea.services.ngcloud.ru/Nail/tf-iot-producer |
+| tf-iot-consumer | https://gitea.services.ngcloud.ru/Nail/tf-iot-consumer |
+
+## Producer (tf-iot-producer)
+
+`app.py` — Flask + KafkaProducer:
+- Генерирует фейковые IoT-события (8 датчиков: температура, влажность, энергия)
+- Шлёт в Kafka каждые `PRODUCE_INTERVAL` секунд (по умолчанию 3)
+- Топик и брокеры — из env (задаёт Terraform)
+- SASL_PLAINTEXT аутентификация
+- `GET /` — статистика (events_sent, errors)
+- `GET /health` — healthcheck
+
+## Consumer (tf-iot-consumer)
+
+`app.py` — Flask + KafkaConsumer + ClickHouse:
+- Читает события из Kafka, пишет батчами по 10 в ClickHouse
+- Создаёт таблицу `iot_events` (MergeTree) при старте
+- SASL_PLAINTEXT → Kafka, HTTP → ClickHouse
+- `GET /` — статистика (events_consumed, events_in_db)
+- `GET /health` — healthcheck

apps/iot-demo/HISTORY/03-terraform.md

@@ -0,0 +1,45 @@
+# Шаг 3 — Terraform-конфиги
+
+**Дата:** 2026-07-21
+
+## Созданы файлы
+
+| Файл | Назначение |
+|------|-----------|
+| `main.tf` | Provider, variables |
+| `locals.tf` | Все настраиваемые параметры |
+| `kafka.tf` | `nubes_kafka` + topic + user |
+| `clickhouse.tf` | `nubes_clickhouse` + database + user |
+| `superset.tf` | `nubes_superset` |
+| `producer.tf` | `nubes_flask` — IoT продюсер |
+| `consumer.tf` | `nubes_flask` — IoT consumer |
+| `terraform.tfvars.example` | Пример переменных |
+
+## Ресурсы (5 штук)
+
+```
+nubes_kafka.main
+  ├── nubes_kafka_topic.events   (iot-events)
+  └── nubes_kafka_user.app       (iot-app, read+write)
+
+nubes_clickhouse.main
+  ├── nubes_clickhouse_database.iot  (iot)
+  └── nubes_clickhouse_user.writer   (iot_writer)
+
+nubes_superset.dash
+
+nubes_flask.producer  → Kafka
+nubes_flask.consumer  → Kafka → ClickHouse
+```
+
+## Передача секретов
+
+- Kafka: `vault_secrets["users"]` → пароль пользователя
+- ClickHouse: `vault_secrets["users"]` → пароль пользователя
+- Bootstrap servers: `state_out_flat["bootstrapServers"]`
+- ClickHouse host: `state_out_flat["internalMaster"]`
+
+## ⚠️ Требует проверки
+
+- Точные ключи `state_out_flat` для Kafka (`bootstrapServers`) и ClickHouse (`internalMaster`) — нужна верификация на реальном стенде
+- Формат `vault_secrets["users"]` для Kafka и ClickHouse — аналогичен PostgreSQL, но требует проверки

apps/iot-demo/HISTORY/04-plan-success.md

@@ -0,0 +1,41 @@
+# Шаг 4 — Terraform plan (успех)
+
+**Дата:** 2026-07-21
+
+## Результат
+
+`terraform plan` — **успешно**. 9 ресурсов, 0 ошибок.
+
+## Исправления
+
+1. **Синтаксис vault_secrets** — `${local.var}` нельзя использовать после точки.
+   Решение: декодировать в отдельный local, хардкодить username (как в CRUD):
+   ```hcl
+   locals {
+     kafka_secrets = jsondecode(nubes_kafka.main.vault_secrets["users"])
+     password      = nonsensitive(local.kafka_secrets["iot-app"].password)
+   }
+   ```
+
+2. **ClickHouse обязательные блоки** — добавлены:
+   - `clickhouse_keeper_configuration` (cpu/memory/replicas)
+   - `backup_configuration` (s3_uid/retain/schedule)
+   - `autoscale_configuration` (enabled/percent/quota/schedule)
+
+## План (9 ресурсов)
+
+| # | Ресурс | Имя |
+|---|--------|-----|
+| 1 | nubes_kafka | iot-kafka |
+| 2 | nubes_kafka_topic | iot-events |
+| 3 | nubes_kafka_user | iot-app |
+| 4 | nubes_clickhouse | iot-clickhouse |
+| 5 | nubes_clickhouse_database | iot |
+| 6 | nubes_clickhouse_user | iot_writer |
+| 7 | nubes_flask | iot-producer |
+| 8 | nubes_flask | iot-consumer |
+| 9 | nubes_superset | iot-superset |
+
+## ⚠️ Остаётся неизвестным
+
+Ключи `state_out_flat` (`bootstrapServers`, `internalMaster`) и формат `vault_secrets` — станут известны только после `apply`. Завёрнуты в `try()` чтобы не упасть, но реальные значения будут пустыми до создания инстансов.

apps/iot-demo/HISTORY/05-test-stand-fail.md

@@ -0,0 +1,37 @@
+# Шаг 5 — Apply на тестовом стенде (неудача)
+
+**Дата:** 2026-07-21
+
+## Результат
+
+Тестовый стенд (`k8s-3` и `k8s-4`) неработоспособен — инфраструктурные проблемы.
+
+## Что создалось
+
+| Ресурс | Статус |
+|--------|--------|
+| nubes_kafka.main (iot-kafka) | ✅ Создан |
+| nubes_kafka_topic.events | ✅ Создан |
+| nubes_kafka_user.app | ❌ operations формат |
+| nubes_clickhouse.main (iot-ch) | ❌ helm.altinity.com timeout |
+| nubes_superset.dash | ❌ IAM 408 timeout |
+| nubes_flask.producer | ❌ pod не запустился |
+| nubes_flask.consumer | ❌ (зависит от CH) |
+
+## Ошибки
+
+1. **ClickHouse**: GitHub/Helm repo недоступен — `context deadline exceeded`
+2. **Superset**: IAM error 408 — auth-api-test таймаутит
+3. **Flask producer**: поды не стартуют — `jlib.k8s [correctReplicaActive] ERROR`
+4. **Kafka user**: `operations` должен быть `Create,Describe,Read,Write` (исправлено)
+
+## Исправления в коде
+
+- `producer.tf` / `consumer.tf`: `vault_secrets` обёрнуты в `try()` с fallback на `"secrets"`
+- `kafka.tf`: `operations = "Create,Describe,Read,Write"` (было `"read,write"`)
+- `locals.tf`: `ch_resource_name = "iot-ch"` (было `"iot-clickhouse"`)
+- `terraform.tfvars`: `realm = "k8s-4-sandbox-nubes-ru"`
+
+## Решение
+
+Перенос на DEV-стенд.