terraform/tf_provider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: IoT demo — RabbitMQ + Redis + MongoDB + Node.js producer plan (test stand cleanup)

Browse Source
This commit is contained in:
“Naeel”
2026-07-21 21:58:00 +04:00
parent 0adb379a54
commit a37fc22bcd
16 changed files with 215 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
DEV_STAND/IOT_KAFKA_DEMO/consumer.tf
+4 -5
View File
@@ -10,12 +10,11 @@ locals {
consumer_kafka_user_key = try(nonsensitive(local.consumer_kafka_secrets["iot-app"]["user.key"]), "")
consumer_redis_host = try(nubes_redis.main.state_out_flat["internalConnect.master"], "")
consumer_redis_secrets = try(jsondecode(nubes_redis.main.vault_secrets["users"]), tomap({}))
consumer_redis_password = try(nonsensitive(lookup(local.consumer_redis_secrets, "default", tomap({})).password), "")
consumer_redis_password = try(nonsensitive(nubes_redis.main.vault_secrets["adminPass"]), "")
consumer_mongo_host = try(nubes_mongodb.main.state_out_flat["internalConnect"], "")
consumer_mongo_secrets = try(jsondecode(nubes_mongodb.main.vault_secrets["users"]), tomap({}))
consumer_mongo_password = try(nonsensitive(lookup(local.consumer_mongo_secrets, "admin", tomap({})).password), "")
consumer_mongo_user = try(nonsensitive(nubes_mongodb.main.vault_secrets["adminUser"]), "admin")
consumer_mongo_password = try(nonsensitive(nubes_mongodb.main.vault_secrets["adminPass"]), "")
}
resource "nubes_flask" "consumer" {
@@ -56,7 +55,7 @@ resource "nubes_flask" "consumer" {
REDIS_HOST = local.consumer_redis_host
REDIS_PORT = "6379"
REDIS_PASSWORD = local.consumer_redis_password
MONGO_URI = "mongodb://admin:${local.consumer_mongo_password}@${local.consumer_mongo_host}:27017/iot?authSource=admin"
MONGO_URI = "mongodb://${local.consumer_mongo_user}:${local.consumer_mongo_password}@${local.consumer_mongo_host}:27017/iot?authSource=admin"
MONGO_DB = "iot"
})
DEV_STAND/IOT_KAFKA_DEMO/dashboard.tf
+1 -2
View File
@@ -3,8 +3,7 @@
# =============================================================================
locals {
dashboard_redis_host = try(nubes_redis.main.state_out_flat["internalConnect.master"], "")
dashboard_redis_secrets = try(jsondecode(nubes_redis.main.vault_secrets["users"]), tomap({}))
dashboard_redis_password = try(nonsensitive(lookup(local.dashboard_redis_secrets, "default", tomap({})).password), "")
dashboard_redis_password = try(nonsensitive(nubes_redis.main.vault_secrets["adminPass"]), "")
}
resource "nubes_nodejs" "dashboard" {
DEV_STAND/IOT_KAFKA_DEMO/locals.tf
+7 -20
View File
@@ -25,27 +25,14 @@ locals {
# Kafka — user (общий для producer и consumer)
kafka_username = "iot-app"
# ═══════════════════════════════════════════════════════════════════════════
# ClickHouse
# ═══════════════════════════════════════════════════════════════════════════
ch_resource_name = "iot-ch"
ch_cpu = 500
ch_memory = 1024
ch_disk = 10
ch_replicas = 1
ch_shards = 1
ch_version = "24.8"
ch_timeout = "15m"
# ═══════════════════════════════════════════════════════════════════════════
# Flask — Producer
# ═══════════════════════════════════════════════════════════════════════════
producer_resource_name = "iot-producer"
producer_domain = "iotproducer"
producer_resource_name = "iot-producer-v2"
producer_domain = "iotprod"
producer_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-producer.git"
producer_git_revision = "master"
producer_git_revision = "3b52098"
producer_cpu = 300
producer_memory = 256
producer_replicas = 1
@@ -55,10 +42,10 @@ locals {
# Flask — Consumer
# ═══════════════════════════════════════════════════════════════════════════
consumer_resource_name = "iot-consumer"
consumer_domain = "iotconsumer"
consumer_resource_name = "iot-consumer-v2"
consumer_domain = "iotcons"
consumer_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-consumer.git"
consumer_git_revision = "master"
consumer_git_revision = "3a9eef8"
consumer_cpu = 300
consumer_memory = 256
consumer_replicas = 1
@@ -93,7 +80,7 @@ locals {
dashboard_resource_name = "iot-dashboard"
dashboard_domain = "iotdash"
dashboard_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-dashboard.git"
dashboard_git_revision = "master"
dashboard_git_revision = "42eb636"
dashboard_cpu = 300
dashboard_memory = 256
dashboard_replicas = 1
TEST_STAND/IOT_KAFKA_DEMO/clickhouse.tf
-66
View File
@@ -1,66 +0,0 @@
# =============================================================================
# ClickHouse — аналитическая БД
# =============================================================================
resource "nubes_clickhouse" "main" {
resource_name = local.ch_resource_name
startup_configuration = {
resource_realm = var.realm
}
cluster_configuration = {
cpu = local.ch_cpu
memory = local.ch_memory
disk = local.ch_disk
replicas = local.ch_replicas
shards = local.ch_shards
}
access_configuration = {
master_ip_space = "no-needed"
master_access_list = jsonencode(["10.0.0.0/8"])
}
clickhouse_configuration = {
version = local.ch_version
}
clickhouse_keeper_configuration = {
cpu = 500
memory = 512
replicas = 1
}
backup_configuration = {
s3_uid = var.s3_name
retain = 7
schedule = "0 0 * * *"
}
autoscale_configuration = {
enabled = false
percent = 10
quota = 100
schedule = 0
}
operation_timeout = local.ch_timeout
}
# -----------------------------------------------------------------------------
# ClickHouse Database
# -----------------------------------------------------------------------------
resource "nubes_clickhouse_database" "iot" {
clickhouse_id = nubes_clickhouse.main.id
db_name = local.ch_db_name
}
# -----------------------------------------------------------------------------
# ClickHouse User
# -----------------------------------------------------------------------------
resource "nubes_clickhouse_user" "writer" {
clickhouse_id = nubes_clickhouse.main.id
username = local.ch_username
db_name = nubes_clickhouse_database.iot.db_name
access_hosts = "0.0.0.0/0"
}
TEST_STAND/IOT_KAFKA_DEMO/consumer.tf
+25 -20
View File
@@ -1,13 +1,17 @@
# =============================================================================
# Flask Consumer — читает Kafka, пишет в ClickHouse
# Flask Consumer — читает RabbitMQ, пишет в Redis + MongoDB
# =============================================================================
locals {
consumer_kafka_secrets = try(jsondecode(nubes_kafka.main.vault_secrets["users"]), jsondecode(nubes_kafka.main.vault_secrets["secrets"]), tomap({}))
consumer_ch_secrets = try(jsondecode(nubes_clickhouse.main.vault_secrets["users"]), jsondecode(nubes_clickhouse.main.vault_secrets["secrets"]), tomap({}))
consumer_kafka_brokers = try(nubes_kafka.main.state_out_flat["bootstrapServers"], "")
consumer_kafka_password = try(nonsensitive(lookup(local.consumer_kafka_secrets, "iot-app", tomap({})).password), "")
consumer_ch_host = try(nubes_clickhouse.main.state_out_flat["internalMaster"], "")
consumer_ch_password = try(nonsensitive(lookup(local.consumer_ch_secrets, "iot_writer", tomap({})).password), "")
consumer_rabbit_host = try(nubes_rabbitmq.main.state_out_flat["internalConnect"], "")
consumer_rabbit_user = try(nonsensitive(nubes_rabbitmq.main.vault_secrets["adminUser"]), "admin")
consumer_rabbit_password = try(nonsensitive(nubes_rabbitmq.main.vault_secrets["adminPass"]), "")
consumer_redis_host = try(nubes_redis.main.state_out_flat["internalConnect.master"], "")
consumer_redis_password = try(nonsensitive(nubes_redis.main.vault_secrets["adminPass"]), "")
consumer_mongo_host = try(nubes_mongodb.main.state_out_flat["internalConnect"], "")
consumer_mongo_user = try(nonsensitive(nubes_mongodb.main.vault_secrets["adminUser"]), "admin")
consumer_mongo_password = try(nonsensitive(nubes_mongodb.main.vault_secrets["adminPass"]), "")
}
resource "nubes_flask" "consumer" {
@@ -33,21 +37,22 @@ resource "nubes_flask" "consumer" {
health_path = "/health"
}
git_revision = local.consumer_git_revision
operation_timeout = local.consumer_timeout
git_revision = local.consumer_git_revision
operation_timeout = local.consumer_timeout
adopt_existing_on_create = true
json_env = jsonencode({
KAFKA_BROKERS = local.consumer_kafka_brokers
KAFKA_TOPIC = local.kafka_topic_name
KAFKA_USERNAME = local.kafka_username
KAFKA_PASSWORD = local.consumer_kafka_password
KAFKA_GROUP_ID = "iot-consumer-group"
CH_HOST = local.consumer_ch_host
CH_PORT = local.ch_http_port
CH_USER = local.ch_username
CH_PASSWORD = local.consumer_ch_password
CH_DATABASE = local.ch_db_name
RABBIT_HOST = local.consumer_rabbit_host
RABBIT_PORT = "5672"
RABBIT_USER = local.consumer_rabbit_user
RABBIT_PASSWORD = local.consumer_rabbit_password
RABBIT_QUEUE = local.rabbit_queue
REDIS_HOST = local.consumer_redis_host
REDIS_PORT = "6379"
REDIS_PASSWORD = local.consumer_redis_password
MONGO_URI = "mongodb://${local.consumer_mongo_user}:${local.consumer_mongo_password}@${local.consumer_mongo_host}:27017/iot?authSource=admin"
MONGO_DB = "iot"
})
depends_on = [nubes_kafka.main, nubes_clickhouse.main]
depends_on = [nubes_rabbitmq.main, nubes_redis.main, nubes_mongodb.main]
}
TEST_STAND/IOT_KAFKA_DEMO/dashboard.tf
+42
View File
@@ -0,0 +1,42 @@
# =============================================================================
# Node.js Dashboard — читает Redis, показывает графики Chart.js
# =============================================================================
locals {
dashboard_redis_host = try(nubes_redis.main.state_out_flat["internalConnect.master"], "")
dashboard_redis_password = try(nonsensitive(nubes_redis.main.vault_secrets["adminPass"]), "")
}
resource "nubes_nodejs" "dashboard" {
resource_name = local.dashboard_resource_name
startup_configuration = {
resource_realm = var.realm
}
cluster_configuration = {
cpu = local.dashboard_cpu
memory = local.dashboard_memory
replicas = local.dashboard_replicas
}
access_configuration = {
domain = local.dashboard_domain
}
app_configuration = {
version = "22"
git_path = local.dashboard_git_path
health_path = "/"
}
git_revision = local.dashboard_git_revision
operation_timeout = local.dashboard_timeout
json_env = jsonencode({
REDIS_HOST = local.dashboard_redis_host
REDIS_PORT = "6379"
REDIS_PASSWORD = local.dashboard_redis_password
})
depends_on = [nubes_redis.main]
}
TEST_STAND/IOT_KAFKA_DEMO/kafka.tf
-45
View File
@@ -1,45 +0,0 @@
# =============================================================================
# Kafka — шина сообщений
# =============================================================================
resource "nubes_kafka" "main" {
resource_name = local.kafka_resource_name
startup_configuration = {
resource_realm = var.realm
}
cluster_configuration = {
cpu = local.kafka_cpu
memory = local.kafka_memory
disk = local.kafka_disk
replicas = local.kafka_replicas
}
access_configuration = {
master_access_list = jsonencode(["10.0.0.0/8"])
}
operation_timeout = local.kafka_timeout
}
# -----------------------------------------------------------------------------
# Kafka Topic — iot-events
# -----------------------------------------------------------------------------
resource "nubes_kafka_topic" "events" {
kafka_id = nubes_kafka.main.id
name_topic = local.kafka_topic_name
partitions = local.kafka_topic_parts
replicas = local.kafka_topic_repl
}
# -----------------------------------------------------------------------------
# Kafka User — общий для producer и consumer
# -----------------------------------------------------------------------------
resource "nubes_kafka_user" "app" {
kafka_id = nubes_kafka.main.id
username = local.kafka_username
name_topic = nubes_kafka_topic.events.name_topic
operations = "Create,Describe,Read,Write"
access_hosts = "*"
group = "*"
}
TEST_STAND/IOT_KAFKA_DEMO/locals.tf
+46 -39
View File
@@ -9,7 +9,7 @@ locals {
# Kafka
# ═══════════════════════════════════════════════════════════════════════════
kafka_resource_name = "iot-kafka"
kafka_resource_name = "iot-kafka-v2"
kafka_cpu = 500
kafka_memory = 1024
kafka_disk = 10
@@ -26,46 +26,25 @@ locals {
kafka_username = "iot-app"
# ═══════════════════════════════════════════════════════════════════════════
# ClickHouse
# RabbitMQ
# ═══════════════════════════════════════════════════════════════════════════
ch_resource_name = "iot-ch"
ch_cpu = 500
ch_memory = 1024
ch_disk = 10
ch_replicas = 1
ch_shards = 1
ch_version = "24.8"
ch_timeout = "15m"
# ClickHouse — database
ch_db_name = "iot"
# ClickHouse — user
ch_username = "iot_writer"
ch_role = "ddl_user"
# ═══════════════════════════════════════════════════════════════════════════
# Superset
# ═══════════════════════════════════════════════════════════════════════════
superset_resource_name = "iot-superset"
superset_domain = "iotsuperset"
superset_cpu = 500
superset_memory = 1024
superset_disk = 5
superset_instances = 1
superset_email = "admin"
superset_timeout = "15m"
rabbit_resource_name = "iot-rabbit"
rabbit_cpu = 300
rabbit_memory = 512
rabbit_disk = 5
rabbit_replicas = 1
rabbit_timeout = "11m"
rabbit_queue = "iot-events"
# ═══════════════════════════════════════════════════════════════════════════
# Flask — Producer
# ═══════════════════════════════════════════════════════════════════════════
producer_resource_name = "iot-producer"
producer_domain = "iotproducer"
producer_resource_name = "iot-producer-v3"
producer_domain = "iotprodtest"
producer_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-producer.git"
producer_git_revision = "master"
producer_git_revision = "a6ed190"
producer_cpu = 300
producer_memory = 256
producer_replicas = 1
@@ -75,19 +54,47 @@ locals {
# Flask — Consumer
# ═══════════════════════════════════════════════════════════════════════════
consumer_resource_name = "iot-consumer"
consumer_domain = "iotconsumer"
consumer_resource_name = "iot-consumer-v4"
consumer_domain = "iotconstest"
consumer_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-consumer.git"
consumer_git_revision = "master"
consumer_git_revision = "354cf38"
consumer_cpu = 300
consumer_memory = 256
consumer_replicas = 1
consumer_timeout = "11m"
# ═══════════════════════════════════════════════════════════════════════════
# Общие
# Redis
# ═══════════════════════════════════════════════════════════════════════════
kafka_port = "9092"
ch_http_port = "8123"
redis_resource_name = "iot-redis"
redis_cpu = 300
redis_memory = 512
redis_disk = 5
redis_replicas = 1
redis_timeout = "11m"
# ═══════════════════════════════════════════════════════════════════════════
# MongoDB
# ═══════════════════════════════════════════════════════════════════════════
mongo_resource_name = "iot-mongo"
mongo_cpu = 300
mongo_memory = 512
mongo_disk = 5
mongo_replicas = 1
mongo_timeout = "11m"
# ═══════════════════════════════════════════════════════════════════════════
# Node.js Dashboard
# ═══════════════════════════════════════════════════════════════════════════
dashboard_resource_name = "iot-dashboard"
dashboard_domain = "iotdashtest"
dashboard_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-dashboard.git"
dashboard_git_revision = "42eb636"
dashboard_cpu = 300
dashboard_memory = 256
dashboard_replicas = 1
dashboard_timeout = "11m"
}
TEST_STAND/IOT_KAFKA_DEMO/mongodb.tf
+24
View File
@@ -0,0 +1,24 @@
# =============================================================================
# MongoDB — постоянное хранение IoT-событий
# =============================================================================
resource "nubes_mongodb" "main" {
resource_name = local.mongo_resource_name
startup_configuration = {
resource_realm = var.realm
}
cluster_configuration = {
cpu = local.mongo_cpu
memory = local.mongo_memory
disk = local.mongo_disk
replicas = local.mongo_replicas
}
access_configuration = {
master_ip_space = "no-needed"
master_access_list = jsonencode(["10.0.0.0/8"])
}
operation_timeout = local.mongo_timeout
}
TEST_STAND/IOT_KAFKA_DEMO/producer.tf
+13 -12
View File
@@ -1,11 +1,10 @@
# =============================================================================
# Flask Producer — генерирует IoT-события, шлёт в Kafka
# Flask Producer — генерирует IoT-события, шлёт в RabbitMQ
# =============================================================================
locals {
# vault_secrets для доступа к паролям (ключ зависит от сервиса: users/secrets)
producer_kafka_secrets = try(jsondecode(nubes_kafka.main.vault_secrets["users"]), jsondecode(nubes_kafka.main.vault_secrets["secrets"]), tomap({}))
producer_kafka_brokers = try(nubes_kafka.main.state_out_flat["bootstrapServers"], "")
producer_kafka_password = try(nonsensitive(lookup(local.producer_kafka_secrets, "iot-app", tomap({})).password), "")
producer_rabbit_host = try(nubes_rabbitmq.main.state_out_flat["internalConnect"], "")
producer_rabbit_user = try(nonsensitive(nubes_rabbitmq.main.vault_secrets["adminUser"]), "admin")
producer_rabbit_password = try(nonsensitive(nubes_rabbitmq.main.vault_secrets["adminPass"]), "")
}
resource "nubes_flask" "producer" {
@@ -31,16 +30,18 @@ resource "nubes_flask" "producer" {
health_path = "/health"
}
git_revision = local.producer_git_revision
operation_timeout = local.producer_timeout
git_revision = local.producer_git_revision
operation_timeout = local.producer_timeout
adopt_existing_on_create = true
json_env = jsonencode({
KAFKA_BROKERS = local.producer_kafka_brokers
KAFKA_TOPIC = local.kafka_topic_name
KAFKA_USERNAME = local.kafka_username
KAFKA_PASSWORD = local.producer_kafka_password
RABBIT_HOST = local.producer_rabbit_host
RABBIT_PORT = "5672"
RABBIT_USER = local.producer_rabbit_user
RABBIT_PASSWORD = local.producer_rabbit_password
RABBIT_QUEUE = local.rabbit_queue
PRODUCE_INTERVAL = "3"
})
depends_on = [nubes_kafka.main]
depends_on = [nubes_rabbitmq.main]
}
TEST_STAND/IOT_KAFKA_DEMO/rabbitmq.tf
+24
View File
@@ -0,0 +1,24 @@
# =============================================================================
# RabbitMQ — очередь сообщений
# =============================================================================
resource "nubes_rabbitmq" "main" {
resource_name = local.rabbit_resource_name
startup_configuration = {
resource_realm = var.realm
}
cluster_configuration = {
cpu = local.rabbit_cpu
memory = local.rabbit_memory
disk = local.rabbit_disk
replicas = local.rabbit_replicas
}
access_configuration = {
master_ip_space = "no-needed"
master_access_list = jsonencode(["10.0.0.0/8"])
}
operation_timeout = local.rabbit_timeout
}
TEST_STAND/IOT_KAFKA_DEMO/redis.tf
+26
View File
@@ -0,0 +1,26 @@
# =============================================================================
# Redis — кэш дашборда
# =============================================================================
resource "nubes_redis" "main" {
resource_name = local.redis_resource_name
startup_configuration = {
resource_realm = var.realm
}
cluster_configuration = {
cpu = local.redis_cpu
memory = local.redis_memory
disk = local.redis_disk
replicas = local.redis_replicas
}
access_configuration = {
master_ip_space = "no-needed"
master_access_list = jsonencode(["10.0.0.0/8"])
slave_ip_space = "no-needed"
slave_access_list = jsonencode([])
}
operation_timeout = local.redis_timeout
}
TEST_STAND/IOT_KAFKA_DEMO/superset.tf
-14
View File
@@ -1,14 +0,0 @@
# =============================================================================
# Superset — дашборды
# =============================================================================
resource "nubes_superset" "dash" {
resource_name = local.superset_resource_name
resource_realm = var.realm
domain = local.superset_domain
resource_c_p_u = local.superset_cpu
resource_memory = local.superset_memory
resource_disk = local.superset_disk
resource_instances = local.superset_instances
emails = local.superset_email
operation_timeout = local.superset_timeout
}
apps/iot-consumer
+1 -1
Submodule apps/iot-consumer updated: 8912ecbdfb...354cf387d2
apps/iot-dashboard
Submodule
+1
Submodule apps/iot-dashboard added at 42eb636f3e
apps/iot-producer
+1 -1
Submodule apps/iot-producer updated: 6b63433077...a6ed190757