terraform/tf_provider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: IoT demo final — 6 services (Kafka+Redis+MongoDB+2Flask+NodeJS), removed broken CH/Superset/AKHQ

Browse Source
This commit is contained in:
“Naeel”
2026-07-21 18:32:35 +04:00
parent 55d4109eee
commit 0adb379a54
6 changed files with 129 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DEV_STAND/IOT_KAFKA_DEMO/clickhouse.tf
-66
View File
@@ -1,66 +0,0 @@
# =============================================================================
# ClickHouse — аналитическая БД
# =============================================================================
resource "nubes_clickhouse" "main" {
resource_name = local.ch_resource_name
startup_configuration = {
resource_realm = var.realm
}
cluster_configuration = {
cpu = local.ch_cpu
memory = local.ch_memory
disk = local.ch_disk
replicas = local.ch_replicas
shards = local.ch_shards
}
access_configuration = {
master_ip_space = "no-needed"
master_access_list = jsonencode(["10.0.0.0/8"])
}
clickhouse_configuration = {
version = local.ch_version
}
clickhouse_keeper_configuration = {
cpu = 500
memory = 512
replicas = 1
}
backup_configuration = {
s3_uid = var.s3_name
retain = 7
schedule = "0 0 * * *"
}
autoscale_configuration = {
enabled = false
percent = 10
quota = 100
schedule = 0
}
operation_timeout = local.ch_timeout
}
# -----------------------------------------------------------------------------
# ClickHouse Database
# -----------------------------------------------------------------------------
resource "nubes_clickhouse_database" "iot" {
clickhouse_id = nubes_clickhouse.main.id
db_name = local.ch_db_name
}
# -----------------------------------------------------------------------------
# ClickHouse User
# -----------------------------------------------------------------------------
resource "nubes_clickhouse_user" "writer" {
clickhouse_id = nubes_clickhouse.main.id
username = local.ch_username
db_name = nubes_clickhouse_database.iot.db_name
access_hosts = "0.0.0.0/0"
}
DEV_STAND/IOT_KAFKA_DEMO/consumer.tf
+15 -10
View File
@@ -1,5 +1,5 @@
# =============================================================================
# Flask Consumer — читает Kafka, пишет в ClickHouse
# Flask Consumer — читает Kafka, пишет в Redis + MongoDB
# =============================================================================
locals {
consumer_kafka_secrets = jsondecode(nubes_kafka.main.vault_secrets["users"])
@@ -8,9 +8,14 @@ locals {
consumer_kafka_ca_crt = try(nonsensitive(local.consumer_kafka_secrets["ca"]["ca.crt"]), "")
consumer_kafka_user_crt = try(nonsensitive(local.consumer_kafka_secrets["iot-app"]["user.crt"]), "")
consumer_kafka_user_key = try(nonsensitive(local.consumer_kafka_secrets["iot-app"]["user.key"]), "")
consumer_ch_secrets = try(jsondecode(nubes_clickhouse.main.vault_secrets["users"]), tomap({}))
consumer_ch_host = try(nubes_clickhouse.main.state_out_flat["internalMaster"], "")
consumer_ch_password = try(nonsensitive(lookup(local.consumer_ch_secrets, "iot_writer", tomap({})).password), "")
consumer_redis_host = try(nubes_redis.main.state_out_flat["internalConnect.master"], "")
consumer_redis_secrets = try(jsondecode(nubes_redis.main.vault_secrets["users"]), tomap({}))
consumer_redis_password = try(nonsensitive(lookup(local.consumer_redis_secrets, "default", tomap({})).password), "")
consumer_mongo_host = try(nubes_mongodb.main.state_out_flat["internalConnect"], "")
consumer_mongo_secrets = try(jsondecode(nubes_mongodb.main.vault_secrets["users"]), tomap({}))
consumer_mongo_password = try(nonsensitive(lookup(local.consumer_mongo_secrets, "admin", tomap({})).password), "")
}
resource "nubes_flask" "consumer" {
@@ -48,12 +53,12 @@ resource "nubes_flask" "consumer" {
KAFKA_USER_CRT = local.consumer_kafka_user_crt
KAFKA_USER_KEY = local.consumer_kafka_user_key
KAFKA_GROUP_ID = "iot-consumer-group"
CH_HOST = local.consumer_ch_host
CH_PORT = local.ch_http_port
CH_USER = local.ch_username
CH_PASSWORD = local.consumer_ch_password
CH_DATABASE = local.ch_db_name
REDIS_HOST = local.consumer_redis_host
REDIS_PORT = "6379"
REDIS_PASSWORD = local.consumer_redis_password
MONGO_URI = "mongodb://admin:${local.consumer_mongo_password}@${local.consumer_mongo_host}:27017/iot?authSource=admin"
MONGO_DB = "iot"
})
depends_on = [nubes_kafka.main, nubes_clickhouse.main]
depends_on = [nubes_kafka.main, nubes_redis.main, nubes_mongodb.main]
}
DEV_STAND/IOT_KAFKA_DEMO/dashboard.tf
+43
View File
@@ -0,0 +1,43 @@
# =============================================================================
# Node.js Dashboard — читает Redis, показывает графики Chart.js
# =============================================================================
locals {
dashboard_redis_host = try(nubes_redis.main.state_out_flat["internalConnect.master"], "")
dashboard_redis_secrets = try(jsondecode(nubes_redis.main.vault_secrets["users"]), tomap({}))
dashboard_redis_password = try(nonsensitive(lookup(local.dashboard_redis_secrets, "default", tomap({})).password), "")
}
resource "nubes_nodejs" "dashboard" {
resource_name = local.dashboard_resource_name
startup_configuration = {
resource_realm = var.realm
}
cluster_configuration = {
cpu = local.dashboard_cpu
memory = local.dashboard_memory
replicas = local.dashboard_replicas
}
access_configuration = {
domain = local.dashboard_domain
}
app_configuration = {
version = "22"
git_path = local.dashboard_git_path
health_path = "/"
}
git_revision = local.dashboard_git_revision
operation_timeout = local.dashboard_timeout
json_env = jsonencode({
REDIS_HOST = local.dashboard_redis_host
REDIS_PORT = "6379"
REDIS_PASSWORD = local.dashboard_redis_password
})
depends_on = [nubes_redis.main]
}
DEV_STAND/IOT_KAFKA_DEMO/locals.tf
+13 -27
View File
@@ -38,26 +38,6 @@ locals {
ch_version = "24.8"
ch_timeout = "15m"
# ClickHouse — database
ch_db_name = "iot"
# ClickHouse — user
ch_username = "iot_writer"
ch_role = "ddl_user"
# ═══════════════════════════════════════════════════════════════════════════
# Superset
# ═══════════════════════════════════════════════════════════════════════════
superset_resource_name = "iot-superset"
superset_domain = "iotsuperset"
superset_cpu = 500
superset_memory = 1024
superset_disk = 5
superset_instances = 1
superset_email = "admin"
superset_timeout = "15m"
# ═══════════════════════════════════════════════════════════════════════════
# Flask — Producer
# ═══════════════════════════════════════════════════════════════════════════
@@ -84,13 +64,6 @@ locals {
consumer_replicas = 1
consumer_timeout = "11m"
# ═══════════════════════════════════════════════════════════════════════════
# Общие
# ═══════════════════════════════════════════════════════════════════════════
kafka_port = "9092"
ch_http_port = "8123"
# ═══════════════════════════════════════════════════════════════════════════
# Redis
# ═══════════════════════════════════════════════════════════════════════════
@@ -112,4 +85,17 @@ locals {
mongo_disk = 5
mongo_replicas = 1
mongo_timeout = "11m"
# ═══════════════════════════════════════════════════════════════════════════
# Node.js Dashboard
# ═══════════════════════════════════════════════════════════════════════════
dashboard_resource_name = "iot-dashboard"
dashboard_domain = "iotdash"
dashboard_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-dashboard.git"
dashboard_git_revision = "master"
dashboard_cpu = 300
dashboard_memory = 256
dashboard_replicas = 1
dashboard_timeout = "11m"
}
DEV_STAND/IOT_KAFKA_DEMO/superset.tf
-14
View File
@@ -1,14 +0,0 @@
# =============================================================================
# Superset — дашборды
# =============================================================================
resource "nubes_superset" "dash" {
resource_name = local.superset_resource_name
resource_realm = var.realm
domain = local.superset_domain
resource_c_p_u = local.superset_cpu
resource_memory = local.superset_memory
resource_disk = local.superset_disk
resource_instances = local.superset_instances
emails = local.superset_email
operation_timeout = local.superset_timeout
}
apps/iot-demo/HISTORY/09-final-deploy.md
+58
View File
@@ -0,0 +1,58 @@
# Шаг 9 — DEV стенд: все 6 сервисов развёрнуты
**Дата:** 2026-07-21
## Финальный состав (6 сервисов)
| # | Ресурс Terraform | Имя | Роль |
|---|-----------------|------|------|
| 1 | `nubes_kafka` | iot-kafka | Шина сообщений |
| 2 | `nubes_redis` | iot-redis | Кэш дашборда |
| 3 | `nubes_mongodb` | iot-mongo | Постоянное хранение |
| 4 | `nubes_flask` | iot-producer | Генератор IoT → Kafka |
| 5 | `nubes_flask` | iot-consumer | Kafka → Redis + MongoDB |
| 6 | `nubes_nodejs` | iot-dashboard | Redis → графики Chart.js |
## Git-репозитории
| Репо | Назначение |
|------|-----------|
| `Nail/tf-iot-producer` | Flask — IoT продюсер (Kafka TLS) |
| `Nail/tf-iot-consumer` | Flask — Kafka → Redis + MongoDB |
| `Nail/tf-iot-dashboard` | Node.js — дашборд Chart.js |
## Поток данных
```
Producer (Flask) → Kafka → Consumer (Flask) → Redis (кэш)
→ MongoDB (история)
Dashboard (Node.js) ← Redis (API /api/latest, /api/counters, /api/recent)
```
## Реальные ключи (определены опытным путём)
### Kafka
- bootstrap: `state_out_flat["internalConnect.bootstrap.fqdn"]:9093` — **TLS**
- vault: `vault_secrets["users"]["iot-app"]["user.password"]`
### Redis
- host: `state_out_flat["internalConnect.master"]`
- vault: `vault_secrets["users"]["default"]["password"]`
### MongoDB
- host: `state_out_flat["internalConnect"]`
- port: 27017
- vault: `vault_secrets["users"]["admin"]["password"]`
## Структура приложений
- **Flask**: `site/app.py` + `requirements.txt` (БЕЗ Procfile)
- **Node.js**: `server.js` + `package.json` (`npm start`)
## Неудачные сервисы
| Сервис | Причина |
|--------|--------|
| `nubes_clickhouse` | GitHub release assets timeout |
| `nubes_superset` | IAM 408 timeout |
| `nubes_akhq` | `toBoolean() on null` — баг платформы |