feat: IoT Kafka demo — Terraform + history (test stand fail, moving to dev)

2026-07-21 17:33:03 +04:00
parent 399745c177
commit 336e6e3795
13 changed files with 536 additions and 0 deletions
@@ -0,0 +1,66 @@
+# =============================================================================
+# ClickHouse — аналитическая БД
+# =============================================================================
+resource "nubes_clickhouse" "main" {
+  resource_name = local.ch_resource_name
+
+  startup_configuration = {
+    resource_realm = var.realm
+  }
+
+  cluster_configuration = {
+    cpu      = local.ch_cpu
+    memory   = local.ch_memory
+    disk     = local.ch_disk
+    replicas = local.ch_replicas
+    shards   = local.ch_shards
+  }
+
+  access_configuration = {
+    master_ip_space    = "no-needed"
+    master_access_list = jsonencode(["10.0.0.0/8"])
+  }
+
+  clickhouse_configuration = {
+    version = local.ch_version
+  }
+
+  clickhouse_keeper_configuration = {
+    cpu      = 500
+    memory   = 512
+    replicas = 1
+  }
+
+  backup_configuration = {
+    s3_uid   = var.s3_name
+    retain   = 7
+    schedule = "0 0 * * *"
+  }
+
+  autoscale_configuration = {
+    enabled  = false
+    percent  = 10
+    quota    = 100
+    schedule = 0
+  }
+
+  operation_timeout = local.ch_timeout
+}
+
+# -----------------------------------------------------------------------------
+# ClickHouse Database
+# -----------------------------------------------------------------------------
+resource "nubes_clickhouse_database" "iot" {
+  clickhouse_id = nubes_clickhouse.main.id
+  db_name       = local.ch_db_name
+}
+
+# -----------------------------------------------------------------------------
+# ClickHouse User
+# -----------------------------------------------------------------------------
+resource "nubes_clickhouse_user" "writer" {
+  clickhouse_id = nubes_clickhouse.main.id
+  username      = local.ch_username
+  db_name       = nubes_clickhouse_database.iot.db_name
+  access_hosts  = "0.0.0.0/0"
+}
@@ -0,0 +1,53 @@
+# =============================================================================
+# Flask Consumer — читает Kafka, пишет в ClickHouse
+# =============================================================================
+locals {
+  consumer_kafka_secrets = try(jsondecode(nubes_kafka.main.vault_secrets["users"]), jsondecode(nubes_kafka.main.vault_secrets["secrets"]), tomap({}))
+  consumer_ch_secrets    = try(jsondecode(nubes_clickhouse.main.vault_secrets["users"]), jsondecode(nubes_clickhouse.main.vault_secrets["secrets"]), tomap({}))
+  consumer_kafka_brokers = try(nubes_kafka.main.state_out_flat["bootstrapServers"], "")
+  consumer_kafka_password = try(nonsensitive(lookup(local.consumer_kafka_secrets, "iot-app", tomap({})).password), "")
+  consumer_ch_host     = try(nubes_clickhouse.main.state_out_flat["internalMaster"], "")
+  consumer_ch_password = try(nonsensitive(lookup(local.consumer_ch_secrets, "iot_writer", tomap({})).password), "")
+}
+
+resource "nubes_flask" "consumer" {
+  resource_name = local.consumer_resource_name
+
+  startup_configuration = {
+    resource_realm = var.realm
+  }
+
+  cluster_configuration = {
+    cpu      = local.consumer_cpu
+    memory   = local.consumer_memory
+    replicas = local.consumer_replicas
+  }
+
+  access_configuration = {
+    domain = local.consumer_domain
+  }
+
+  app_configuration = {
+    version     = "3.12"
+    git_path    = local.consumer_git_path
+    health_path = "/health"
+  }
+
+  git_revision      = local.consumer_git_revision
+  operation_timeout = local.consumer_timeout
+
+  json_env = jsonencode({
+    KAFKA_BROKERS  = local.consumer_kafka_brokers
+    KAFKA_TOPIC    = local.kafka_topic_name
+    KAFKA_USERNAME = local.kafka_username
+    KAFKA_PASSWORD = local.consumer_kafka_password
+    KAFKA_GROUP_ID = "iot-consumer-group"
+    CH_HOST        = local.consumer_ch_host
+    CH_PORT        = local.ch_http_port
+    CH_USER        = local.ch_username
+    CH_PASSWORD    = local.consumer_ch_password
+    CH_DATABASE    = local.ch_db_name
+  })
+
+  depends_on = [nubes_kafka.main, nubes_clickhouse.main]
+}
@@ -0,0 +1,45 @@
+# =============================================================================
+# Kafka — шина сообщений
+# =============================================================================
+resource "nubes_kafka" "main" {
+  resource_name = local.kafka_resource_name
+
+  startup_configuration = {
+    resource_realm = var.realm
+  }
+
+  cluster_configuration = {
+    cpu      = local.kafka_cpu
+    memory   = local.kafka_memory
+    disk     = local.kafka_disk
+    replicas = local.kafka_replicas
+  }
+
+  access_configuration = {
+    master_access_list = jsonencode(["10.0.0.0/8"])
+  }
+
+  operation_timeout = local.kafka_timeout
+}
+
+# -----------------------------------------------------------------------------
+# Kafka Topic — iot-events
+# -----------------------------------------------------------------------------
+resource "nubes_kafka_topic" "events" {
+  kafka_id   = nubes_kafka.main.id
+  name_topic = local.kafka_topic_name
+  partitions = local.kafka_topic_parts
+  replicas   = local.kafka_topic_repl
+}
+
+# -----------------------------------------------------------------------------
+# Kafka User — общий для producer и consumer
+# -----------------------------------------------------------------------------
+resource "nubes_kafka_user" "app" {
+  kafka_id     = nubes_kafka.main.id
+  username     = local.kafka_username
+  name_topic   = nubes_kafka_topic.events.name_topic
+  operations   = "Create,Describe,Read,Write"
+  access_hosts = "*"
+  group        = "*"
+}
@@ -0,0 +1,93 @@
+# =============================================================================
+# locals.tf — все настраиваемые значения IOT_KAFKA_DEMO
+# Kafka + ClickHouse + Superset + Flask (producer + consumer)
+# =============================================================================
+
+locals {
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Kafka
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  kafka_resource_name = "iot-kafka"
+  kafka_cpu           = 500
+  kafka_memory        = 1024
+  kafka_disk          = 10
+  kafka_replicas      = 1
+  kafka_version       = "3.7"
+  kafka_timeout       = "15m"
+
+  # Kafka — topic
+  kafka_topic_name  = "iot-events"
+  kafka_topic_parts = 3
+  kafka_topic_repl  = 1
+
+  # Kafka — user (общий для producer и consumer)
+  kafka_username = "iot-app"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # ClickHouse
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  ch_resource_name = "iot-ch"
+  ch_cpu           = 500
+  ch_memory        = 1024
+  ch_disk          = 10
+  ch_replicas      = 1
+  ch_shards        = 1
+  ch_version       = "24.8"
+  ch_timeout       = "15m"
+
+  # ClickHouse — database
+  ch_db_name = "iot"
+
+  # ClickHouse — user
+  ch_username = "iot_writer"
+  ch_role     = "ddl_user"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Superset
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  superset_resource_name = "iot-superset"
+  superset_domain        = "iotsuperset"
+  superset_cpu           = 500
+  superset_memory        = 1024
+  superset_disk          = 5
+  superset_instances     = 1
+  superset_email         = "admin"
+  superset_timeout       = "15m"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Flask — Producer
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  producer_resource_name = "iot-producer"
+  producer_domain        = "iotproducer"
+  producer_git_path      = "https://gitea.services.ngcloud.ru/Nail/tf-iot-producer.git"
+  producer_git_revision  = "master"
+  producer_cpu           = 300
+  producer_memory        = 256
+  producer_replicas      = 1
+  producer_timeout       = "11m"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Flask — Consumer
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  consumer_resource_name = "iot-consumer"
+  consumer_domain        = "iotconsumer"
+  consumer_git_path      = "https://gitea.services.ngcloud.ru/Nail/tf-iot-consumer.git"
+  consumer_git_revision  = "master"
+  consumer_cpu           = 300
+  consumer_memory        = 256
+  consumer_replicas      = 1
+  consumer_timeout       = "11m"
+
+  # ═══════════════════════════════════════════════════════════════════════════
+  # Общие
+  # ═══════════════════════════════════════════════════════════════════════════
+
+  kafka_port   = "9092"
+  ch_http_port = "8123"
+}
@@ -0,0 +1,29 @@
+terraform {
+  required_providers {
+    nubes = {
+      source  = "registry.kube5s.ru/nubes-test/nubes"
+      version = "5.1.16"
+    }
+  }
+}
+
+variable "api_token" {
+  type        = string
+  sensitive   = true
+  description = "Nubes API token"
+}
+
+variable "realm" {
+  type        = string
+  description = "resource_realm parameter for all resources"
+}
+
+variable "s3_name" {
+  type        = string
+  description = "S3 user name for backups"
+}
+
+provider "nubes" {
+  api_token    = var.api_token
+  api_endpoint = "https://lk-api-gateway-test.ngcloud.ru/api/v1/svc"
+}
@@ -0,0 +1,46 @@
+# =============================================================================
+# Flask Producer — генерирует IoT-события, шлёт в Kafka
+# =============================================================================
+locals {
+  # vault_secrets для доступа к паролям (ключ зависит от сервиса: users/secrets)
+  producer_kafka_secrets = try(jsondecode(nubes_kafka.main.vault_secrets["users"]), jsondecode(nubes_kafka.main.vault_secrets["secrets"]), tomap({}))
+  producer_kafka_brokers = try(nubes_kafka.main.state_out_flat["bootstrapServers"], "")
+  producer_kafka_password = try(nonsensitive(lookup(local.producer_kafka_secrets, "iot-app", tomap({})).password), "")
+}
+
+resource "nubes_flask" "producer" {
+  resource_name = local.producer_resource_name
+
+  startup_configuration = {
+    resource_realm = var.realm
+  }
+
+  cluster_configuration = {
+    cpu      = local.producer_cpu
+    memory   = local.producer_memory
+    replicas = local.producer_replicas
+  }
+
+  access_configuration = {
+    domain = local.producer_domain
+  }
+
+  app_configuration = {
+    version     = "3.12"
+    git_path    = local.producer_git_path
+    health_path = "/health"
+  }
+
+  git_revision      = local.producer_git_revision
+  operation_timeout = local.producer_timeout
+
+  json_env = jsonencode({
+    KAFKA_BROKERS   = local.producer_kafka_brokers
+    KAFKA_TOPIC     = local.kafka_topic_name
+    KAFKA_USERNAME  = local.kafka_username
+    KAFKA_PASSWORD  = local.producer_kafka_password
+    PRODUCE_INTERVAL = "3"
+  })
+
+  depends_on = [nubes_kafka.main]
+}
@@ -0,0 +1,14 @@
+# =============================================================================
+# Superset — дашборды
+# =============================================================================
+resource "nubes_superset" "dash" {
+  resource_name     = local.superset_resource_name
+  resource_realm    = var.realm
+  domain            = local.superset_domain
+  resource_c_p_u    = local.superset_cpu
+  resource_memory   = local.superset_memory
+  resource_disk     = local.superset_disk
+  resource_instances = local.superset_instances
+  emails            = local.superset_email
+  operation_timeout = local.superset_timeout
+}
@@ -0,0 +1,4 @@
+# Скопировать в terraform.tfvars и заполнить
+api_token = "ВАШ_API_ТОКЕН"
+realm     = "ВАШ_REALM"
+s3_name   = "ВАШ_S3_USER"