feat: IoT demo — Kafka + Redis + MongoDB, 6 services, removed broken AKHQ/ClickHouse/Superset
This commit is contained in:
“Naeel”
@@ -0,0 +1,66 @@
|
||||
# =============================================================================
|
||||
# ClickHouse — аналитическая БД
|
||||
# =============================================================================
|
||||
resource "nubes_clickhouse" "main" {
|
||||
resource_name = local.ch_resource_name
|
||||
|
||||
startup_configuration = {
|
||||
resource_realm = var.realm
|
||||
}
|
||||
|
||||
cluster_configuration = {
|
||||
cpu = local.ch_cpu
|
||||
memory = local.ch_memory
|
||||
disk = local.ch_disk
|
||||
replicas = local.ch_replicas
|
||||
shards = local.ch_shards
|
||||
}
|
||||
|
||||
access_configuration = {
|
||||
master_ip_space = "no-needed"
|
||||
master_access_list = jsonencode(["10.0.0.0/8"])
|
||||
}
|
||||
|
||||
clickhouse_configuration = {
|
||||
version = local.ch_version
|
||||
}
|
||||
|
||||
clickhouse_keeper_configuration = {
|
||||
cpu = 500
|
||||
memory = 512
|
||||
replicas = 1
|
||||
}
|
||||
|
||||
backup_configuration = {
|
||||
s3_uid = var.s3_name
|
||||
retain = 7
|
||||
schedule = "0 0 * * *"
|
||||
}
|
||||
|
||||
autoscale_configuration = {
|
||||
enabled = false
|
||||
percent = 10
|
||||
quota = 100
|
||||
schedule = 0
|
||||
}
|
||||
|
||||
operation_timeout = local.ch_timeout
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# ClickHouse Database
|
||||
# -----------------------------------------------------------------------------
|
||||
resource "nubes_clickhouse_database" "iot" {
|
||||
clickhouse_id = nubes_clickhouse.main.id
|
||||
db_name = local.ch_db_name
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# ClickHouse User
|
||||
# -----------------------------------------------------------------------------
|
||||
resource "nubes_clickhouse_user" "writer" {
|
||||
clickhouse_id = nubes_clickhouse.main.id
|
||||
username = local.ch_username
|
||||
db_name = nubes_clickhouse_database.iot.db_name
|
||||
access_hosts = "0.0.0.0/0"
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
# =============================================================================
|
||||
# Flask Consumer — читает Kafka, пишет в ClickHouse
|
||||
# =============================================================================
|
||||
locals {
|
||||
consumer_kafka_secrets = jsondecode(nubes_kafka.main.vault_secrets["users"])
|
||||
consumer_kafka_bootstrap = "${try(nubes_kafka.main.state_out_flat["internalConnect.bootstrap.fqdn"], "")}:${try(nubes_kafka.main.state_out_flat["internalConnect.bootstrap.port"], "9093")}"
|
||||
consumer_kafka_password = try(nonsensitive(local.consumer_kafka_secrets["iot-app"]["user.password"]), "")
|
||||
consumer_kafka_ca_crt = try(nonsensitive(local.consumer_kafka_secrets["ca"]["ca.crt"]), "")
|
||||
consumer_kafka_user_crt = try(nonsensitive(local.consumer_kafka_secrets["iot-app"]["user.crt"]), "")
|
||||
consumer_kafka_user_key = try(nonsensitive(local.consumer_kafka_secrets["iot-app"]["user.key"]), "")
|
||||
consumer_ch_secrets = try(jsondecode(nubes_clickhouse.main.vault_secrets["users"]), tomap({}))
|
||||
consumer_ch_host = try(nubes_clickhouse.main.state_out_flat["internalMaster"], "")
|
||||
consumer_ch_password = try(nonsensitive(lookup(local.consumer_ch_secrets, "iot_writer", tomap({})).password), "")
|
||||
}
|
||||
|
||||
resource "nubes_flask" "consumer" {
|
||||
resource_name = local.consumer_resource_name
|
||||
|
||||
startup_configuration = {
|
||||
resource_realm = var.realm
|
||||
}
|
||||
|
||||
cluster_configuration = {
|
||||
cpu = local.consumer_cpu
|
||||
memory = local.consumer_memory
|
||||
replicas = local.consumer_replicas
|
||||
}
|
||||
|
||||
access_configuration = {
|
||||
domain = local.consumer_domain
|
||||
}
|
||||
|
||||
app_configuration = {
|
||||
version = "3.12"
|
||||
git_path = local.consumer_git_path
|
||||
health_path = "/health"
|
||||
}
|
||||
|
||||
git_revision = local.consumer_git_revision
|
||||
operation_timeout = local.consumer_timeout
|
||||
|
||||
json_env = jsonencode({
|
||||
KAFKA_BROKERS = local.consumer_kafka_bootstrap
|
||||
KAFKA_TOPIC = local.kafka_topic_name
|
||||
KAFKA_USERNAME = local.kafka_username
|
||||
KAFKA_PASSWORD = local.consumer_kafka_password
|
||||
KAFKA_CA_CRT = local.consumer_kafka_ca_crt
|
||||
KAFKA_USER_CRT = local.consumer_kafka_user_crt
|
||||
KAFKA_USER_KEY = local.consumer_kafka_user_key
|
||||
KAFKA_GROUP_ID = "iot-consumer-group"
|
||||
CH_HOST = local.consumer_ch_host
|
||||
CH_PORT = local.ch_http_port
|
||||
CH_USER = local.ch_username
|
||||
CH_PASSWORD = local.consumer_ch_password
|
||||
CH_DATABASE = local.ch_db_name
|
||||
})
|
||||
|
||||
depends_on = [nubes_kafka.main, nubes_clickhouse.main]
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
# =============================================================================
|
||||
# Kafka — шина сообщений
|
||||
# =============================================================================
|
||||
resource "nubes_kafka" "main" {
|
||||
resource_name = local.kafka_resource_name
|
||||
|
||||
startup_configuration = {
|
||||
resource_realm = var.realm
|
||||
}
|
||||
|
||||
cluster_configuration = {
|
||||
cpu = local.kafka_cpu
|
||||
memory = local.kafka_memory
|
||||
disk = local.kafka_disk
|
||||
replicas = local.kafka_replicas
|
||||
}
|
||||
|
||||
access_configuration = {
|
||||
master_access_list = jsonencode(["10.0.0.0/8"])
|
||||
}
|
||||
|
||||
operation_timeout = local.kafka_timeout
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Kafka Topic — iot-events
|
||||
# -----------------------------------------------------------------------------
|
||||
resource "nubes_kafka_topic" "events" {
|
||||
kafka_id = nubes_kafka.main.id
|
||||
name_topic = local.kafka_topic_name
|
||||
partitions = local.kafka_topic_parts
|
||||
replicas = local.kafka_topic_repl
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Kafka User — общий для producer и consumer
|
||||
# -----------------------------------------------------------------------------
|
||||
resource "nubes_kafka_user" "app" {
|
||||
kafka_id = nubes_kafka.main.id
|
||||
username = local.kafka_username
|
||||
name_topic = nubes_kafka_topic.events.name_topic
|
||||
operations = "Create,Describe,Read,Write"
|
||||
access_hosts = "*"
|
||||
group = "*"
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
# =============================================================================
|
||||
# locals.tf — все настраиваемые значения IOT_KAFKA_DEMO
|
||||
# Kafka + ClickHouse + Superset + Flask (producer + consumer)
|
||||
# =============================================================================
|
||||
|
||||
locals {
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Kafka
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
kafka_resource_name = "iot-kafka"
|
||||
kafka_cpu = 500
|
||||
kafka_memory = 1024
|
||||
kafka_disk = 10
|
||||
kafka_replicas = 1
|
||||
kafka_version = "3.7"
|
||||
kafka_timeout = "15m"
|
||||
|
||||
# Kafka — topic
|
||||
kafka_topic_name = "iot-events"
|
||||
kafka_topic_parts = 3
|
||||
kafka_topic_repl = 1
|
||||
|
||||
# Kafka — user (общий для producer и consumer)
|
||||
kafka_username = "iot-app"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# ClickHouse
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
ch_resource_name = "iot-ch"
|
||||
ch_cpu = 500
|
||||
ch_memory = 1024
|
||||
ch_disk = 10
|
||||
ch_replicas = 1
|
||||
ch_shards = 1
|
||||
ch_version = "24.8"
|
||||
ch_timeout = "15m"
|
||||
|
||||
# ClickHouse — database
|
||||
ch_db_name = "iot"
|
||||
|
||||
# ClickHouse — user
|
||||
ch_username = "iot_writer"
|
||||
ch_role = "ddl_user"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Superset
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
superset_resource_name = "iot-superset"
|
||||
superset_domain = "iotsuperset"
|
||||
superset_cpu = 500
|
||||
superset_memory = 1024
|
||||
superset_disk = 5
|
||||
superset_instances = 1
|
||||
superset_email = "admin"
|
||||
superset_timeout = "15m"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Flask — Producer
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
producer_resource_name = "iot-producer"
|
||||
producer_domain = "iotproducer"
|
||||
producer_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-producer.git"
|
||||
producer_git_revision = "master"
|
||||
producer_cpu = 300
|
||||
producer_memory = 256
|
||||
producer_replicas = 1
|
||||
producer_timeout = "11m"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Flask — Consumer
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
consumer_resource_name = "iot-consumer"
|
||||
consumer_domain = "iotconsumer"
|
||||
consumer_git_path = "https://gitea.services.ngcloud.ru/Nail/tf-iot-consumer.git"
|
||||
consumer_git_revision = "master"
|
||||
consumer_cpu = 300
|
||||
consumer_memory = 256
|
||||
consumer_replicas = 1
|
||||
consumer_timeout = "11m"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Общие
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
kafka_port = "9092"
|
||||
ch_http_port = "8123"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Redis
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
redis_resource_name = "iot-redis"
|
||||
redis_cpu = 300
|
||||
redis_memory = 512
|
||||
redis_disk = 5
|
||||
redis_replicas = 1
|
||||
redis_timeout = "11m"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# MongoDB
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
mongo_resource_name = "iot-mongo"
|
||||
mongo_cpu = 300
|
||||
mongo_memory = 512
|
||||
mongo_disk = 5
|
||||
mongo_replicas = 1
|
||||
mongo_timeout = "11m"
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
terraform {
|
||||
required_providers {
|
||||
nubes = {
|
||||
source = "registry.kube5s.ru/nubes-test/nubes"
|
||||
version = "5.1.16"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
variable "api_token" {
|
||||
type = string
|
||||
sensitive = true
|
||||
description = "Nubes API token"
|
||||
}
|
||||
|
||||
variable "realm" {
|
||||
type = string
|
||||
description = "resource_realm parameter for all resources"
|
||||
}
|
||||
|
||||
variable "s3_name" {
|
||||
type = string
|
||||
description = "S3 user name for backups"
|
||||
}
|
||||
|
||||
provider "nubes" {
|
||||
api_token = var.api_token
|
||||
api_endpoint = "https://lk-api-gateway-dev.ngcloud.ru/api/v1/svc"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
# =============================================================================
|
||||
# MongoDB — постоянное хранение IoT-событий
|
||||
# =============================================================================
|
||||
resource "nubes_mongodb" "main" {
|
||||
resource_name = local.mongo_resource_name
|
||||
|
||||
startup_configuration = {
|
||||
resource_realm = var.realm
|
||||
}
|
||||
|
||||
cluster_configuration = {
|
||||
cpu = local.mongo_cpu
|
||||
memory = local.mongo_memory
|
||||
disk = local.mongo_disk
|
||||
replicas = local.mongo_replicas
|
||||
}
|
||||
|
||||
access_configuration = {
|
||||
master_ip_space = "no-needed"
|
||||
master_access_list = jsonencode(["10.0.0.0/8"])
|
||||
}
|
||||
|
||||
operation_timeout = local.mongo_timeout
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
# =============================================================================
|
||||
# Flask Producer — генерирует IoT-события, шлёт в Kafka
|
||||
# =============================================================================
|
||||
locals {
|
||||
producer_kafka_secrets = jsondecode(nubes_kafka.main.vault_secrets["users"])
|
||||
producer_kafka_bootstrap = "${try(nubes_kafka.main.state_out_flat["internalConnect.bootstrap.fqdn"], "")}:${try(nubes_kafka.main.state_out_flat["internalConnect.bootstrap.port"], "9093")}"
|
||||
producer_kafka_password = try(nonsensitive(local.producer_kafka_secrets["iot-app"]["user.password"]), "")
|
||||
producer_kafka_ca_crt = try(nonsensitive(local.producer_kafka_secrets["ca"]["ca.crt"]), "")
|
||||
producer_kafka_user_crt = try(nonsensitive(local.producer_kafka_secrets["iot-app"]["user.crt"]), "")
|
||||
producer_kafka_user_key = try(nonsensitive(local.producer_kafka_secrets["iot-app"]["user.key"]), "")
|
||||
}
|
||||
|
||||
resource "nubes_flask" "producer" {
|
||||
resource_name = local.producer_resource_name
|
||||
|
||||
startup_configuration = {
|
||||
resource_realm = var.realm
|
||||
}
|
||||
|
||||
cluster_configuration = {
|
||||
cpu = local.producer_cpu
|
||||
memory = local.producer_memory
|
||||
replicas = local.producer_replicas
|
||||
}
|
||||
|
||||
access_configuration = {
|
||||
domain = local.producer_domain
|
||||
}
|
||||
|
||||
app_configuration = {
|
||||
version = "3.12"
|
||||
git_path = local.producer_git_path
|
||||
health_path = "/health"
|
||||
}
|
||||
|
||||
git_revision = local.producer_git_revision
|
||||
operation_timeout = local.producer_timeout
|
||||
|
||||
json_env = jsonencode({
|
||||
KAFKA_BROKERS = local.producer_kafka_bootstrap
|
||||
KAFKA_TOPIC = local.kafka_topic_name
|
||||
KAFKA_USERNAME = local.kafka_username
|
||||
KAFKA_PASSWORD = local.producer_kafka_password
|
||||
KAFKA_CA_CRT = local.producer_kafka_ca_crt
|
||||
KAFKA_USER_CRT = local.producer_kafka_user_crt
|
||||
KAFKA_USER_KEY = local.producer_kafka_user_key
|
||||
PRODUCE_INTERVAL = "3"
|
||||
})
|
||||
|
||||
depends_on = [nubes_kafka.main]
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
# =============================================================================
|
||||
# Redis — кэш дашборда
|
||||
# =============================================================================
|
||||
resource "nubes_redis" "main" {
|
||||
resource_name = local.redis_resource_name
|
||||
|
||||
startup_configuration = {
|
||||
resource_realm = var.realm
|
||||
}
|
||||
|
||||
cluster_configuration = {
|
||||
cpu = local.redis_cpu
|
||||
memory = local.redis_memory
|
||||
disk = local.redis_disk
|
||||
replicas = local.redis_replicas
|
||||
}
|
||||
|
||||
access_configuration = {
|
||||
master_ip_space = "no-needed"
|
||||
master_access_list = jsonencode(["10.0.0.0/8"])
|
||||
slave_ip_space = "no-needed"
|
||||
slave_access_list = jsonencode([])
|
||||
}
|
||||
|
||||
operation_timeout = local.redis_timeout
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
# =============================================================================
|
||||
# Superset — дашборды
|
||||
# =============================================================================
|
||||
resource "nubes_superset" "dash" {
|
||||
resource_name = local.superset_resource_name
|
||||
resource_realm = var.realm
|
||||
domain = local.superset_domain
|
||||
resource_c_p_u = local.superset_cpu
|
||||
resource_memory = local.superset_memory
|
||||
resource_disk = local.superset_disk
|
||||
resource_instances = local.superset_instances
|
||||
emails = local.superset_email
|
||||
operation_timeout = local.superset_timeout
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
# Скопировать в terraform.tfvars и заполнить
|
||||
api_token = "ВАШ_API_ТОКЕН"
|
||||
realm = "ВАШ_REALM"
|
||||
s3_name = "ВАШ_S3_USER"
|
||||
Reference in New Issue
Block a user