Nail/IoT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: замена Kafka → shared-SQS в IoT pipeline

Browse Source 
- mqtt-bridge: kafka.Writer → SQS SendMessage (AWS SDK Go v2)
- kafka-consumer → sqs-consumer: polling loop с ReceiveMessage/DeleteMessage
- admin stats: Kafka lag → SQS GetQueueAttributes
- Удалён kafka-consumer, добавлен sqs-consumer
- go.mod: убран segmentio/kafka-go, добавлен aws-sdk-go-v2
- Dockerfile, Makefile: kafka-consumer → sqs-consumer
- .gitignore: исправлен чтобы не игнорировать cmd/ директории
- deployments: новый iot-sqs-consumer.yaml, обновлён mqtt-bridge
- doc/decisions: задокументировано решение
This commit is contained in:
Naeel
2026-04-12 15:28:45 +03:00
parent a5e52c9e1c
commit 6caab6b729
13 changed files with 848 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/handler/handler.go
-2
View File
@@ -30,8 +30,6 @@ K8s client.Client
Scheme *runtime.Scheme
// IoTPG — хранилище IoT телеметрии (per-tenant Postgres). nil если IOT_PG_DSN не задан.
IoTPG *iotpg.IoTPostgresStore
// KafkaBrokers — адреса Kafka брокеров для чтения consumer lag на admin странице.
KafkaBrokers string
Log *slog.Logger
}
internal/api/handler/iot_admin_stats_handler.go
+58 -67
View File
@@ -1,4 +1,5 @@
// Создано: 2026-04-06
// Изменено: 2026-04-12 (замена Kafka lag → SQS queue stats)
// iot_admin_stats_handler.go — handler для страницы администратора IoT.
//
// Endpoints:
@@ -6,8 +7,8 @@
//
// Источники данных:
// - PostgreSQL (IoTPG): counts per tenant, last 1h/24h, latest rows
// - Kafka: consumer lag (latest offset - committed offset для group iot-pg-consumer)
// - K8s: статус подов iot-mqtt-bridge и iot-kafka-consumer
// - SQS: approximate message count (ApproximateNumberOfMessages)
// - K8s: статус подов iot-mqtt-bridge и iot-sqs-consumer
//
// Авторизация: Bearer из env ADMIN_STATS_TOKEN.
// Если ADMIN_STATS_TOKEN не задан — endpoint возвращает 503.
@@ -19,10 +20,14 @@ import (
"fmt"
"net/http"
"os"
"strconv"
"strings"
"time"
kafka "github.com/segmentio/kafka-go"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/sqs"
sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types"
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)
@@ -36,12 +41,11 @@ type iotAdminPodStatus struct {
Age string `json:"age"`
}
// iotAdminKafkaStats — информация о Kafka топике и consumer lag.
type iotAdminKafkaStats struct {
LatestOffset int64 `json:"latest_offset"`
CommittedOffset int64 `json:"committed_offset"`
ConsumerLag int64 `json:"consumer_lag"`
Error string `json:"error,omitempty"`
// iotAdminSQSStats — информация о SQS очереди для мониторинга.
type iotAdminSQSStats struct {
ApproximateMessages int64 `json:"approximate_messages"`
ApproximateMessagesNotVisible int64 `json:"approximate_messages_not_visible"`
Error string `json:"error,omitempty"`
}
// AdminStats обрабатывает GET /iot-admin/stats.
@@ -78,8 +82,8 @@ func (h *Handler) AdminStats(w http.ResponseWriter, r *http.Request) {
result["postgres"] = map[string]any{"reachable": false, "error": "IoTPG not configured"}
}
// Kafka: consumer lag для топика iot.telemetry / группы iot-pg-consumer
result["kafka"] = h.collectIotKafkaLag(ctx)
// SQS: approximate message count для очереди iot-telemetry
result["sqs"] = h.collectIotSQSStats(ctx)
// K8s: статус подов bridge и consumer
result["pods"] = h.collectIotPodStatuses(ctx)
@@ -87,72 +91,59 @@ func (h *Handler) AdminStats(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, result)
}
// collectIotKafkaLag получает latest offset топика и committed offset consumer group,
// вычисляет lag = latest - committed.
// Topic: "iot.telemetry", Consumer Group: "iot-pg-consumer".
func (h *Handler) collectIotKafkaLag(ctx context.Context) iotAdminKafkaStats {
if h.KafkaBrokers == "" {
return iotAdminKafkaStats{Error: "KAFKA_BROKERS not configured"}
// collectIotSQSStats получает ApproximateNumberOfMessages из SQS очереди.
// Использует SQS_ENDPOINT, SQS_ACCESS_KEY, SQS_SECRET_KEY, SQS_QUEUE_NAME из env.
func (h *Handler) collectIotSQSStats(ctx context.Context) iotAdminSQSStats {
endpoint := os.Getenv("SQS_ENDPOINT")
accessKey := os.Getenv("SQS_ACCESS_KEY")
secretKey := os.Getenv("SQS_SECRET_KEY")
queueName := os.Getenv("SQS_QUEUE_NAME")
if queueName == "" {
queueName = "iot-telemetry"
}
region := os.Getenv("SQS_REGION")
if region == "" {
region = "us-east-1"
}
brokers := strings.Split(h.KafkaBrokers, ",")
brokerAddr := kafka.TCP(brokers...)
kc := &kafka.Client{
Addr: brokerAddr,
Timeout: 5 * time.Second,
if endpoint == "" || accessKey == "" || secretKey == "" {
return iotAdminSQSStats{Error: "SQS credentials not configured (SQS_ENDPOINT, SQS_ACCESS_KEY, SQS_SECRET_KEY)"}
}
const topic = "iot.telemetry"
const group = "iot-pg-consumer"
sqsClient := sqs.New(sqs.Options{
Region: region,
Credentials: credentials.NewStaticCredentialsProvider(
accessKey, secretKey, "",
),
BaseEndpoint: aws.String(endpoint),
})
// Получаем latest offset (конец лога — сколько всего сообщений прошло)
offsetsResp, err := kc.ListOffsets(ctx, &kafka.ListOffsetsRequest{
Addr: brokerAddr,
Topics: map[string][]kafka.OffsetRequest{
topic: {kafka.LastOffsetOf(0)},
// Получаем URL очереди
queueUrlOut, err := sqsClient.GetQueueUrl(ctx, &sqs.GetQueueUrlInput{
QueueName: aws.String(queueName),
})
if err != nil {
return iotAdminSQSStats{Error: fmt.Sprintf("GetQueueUrl: %v", err)}
}
// Запрашиваем атрибуты очереди — approximate message counts
attrsOut, err := sqsClient.GetQueueAttributes(ctx, &sqs.GetQueueAttributesInput{
QueueUrl: queueUrlOut.QueueUrl,
AttributeNames: []sqstypes.QueueAttributeName{
sqstypes.QueueAttributeNameApproximateNumberOfMessages,
sqstypes.QueueAttributeNameApproximateNumberOfMessagesNotVisible,
},
})
if err != nil {
return iotAdminKafkaStats{Error: fmt.Sprintf("list offsets: %v", err)}
return iotAdminSQSStats{Error: fmt.Sprintf("GetQueueAttributes: %v", err)}
}
var latestOffset int64
if partitions, ok := offsetsResp.Topics[topic]; ok && len(partitions) > 0 {
if partitions[0].Error == nil {
latestOffset = partitions[0].LastOffset
}
}
approxMsg, _ := strconv.ParseInt(attrsOut.Attributes["ApproximateNumberOfMessages"], 10, 64)
approxNotVisible, _ := strconv.ParseInt(attrsOut.Attributes["ApproximateNumberOfMessagesNotVisible"], 10, 64)
// Получаем committed offset consumer group (что consumer уже обработал)
fetchResp, err := kc.OffsetFetch(ctx, &kafka.OffsetFetchRequest{
Addr: brokerAddr,
GroupID: group,
Topics: map[string][]int{topic: {0}},
})
if err != nil {
return iotAdminKafkaStats{
LatestOffset: latestOffset,
Error: fmt.Sprintf("offset fetch: %v", err),
}
}
var committedOffset int64
if partitions, ok := fetchResp.Topics[topic]; ok && len(partitions) > 0 {
if partitions[0].Error == nil {
committedOffset = partitions[0].CommittedOffset
}
}
lag := latestOffset - committedOffset
if lag < 0 {
lag = 0
}
return iotAdminKafkaStats{
LatestOffset: latestOffset,
CommittedOffset: committedOffset,
ConsumerLag: lag,
return iotAdminSQSStats{
ApproximateMessages: approxMsg,
ApproximateMessagesNotVisible: approxNotVisible,
}
}
@@ -160,7 +151,7 @@ func (h *Handler) collectIotKafkaLag(ctx context.Context) iotAdminKafkaStats {
func (h *Handler) collectIotPodStatuses(ctx context.Context) map[string]any {
result := map[string]any{}
for _, appLabel := range []string{"iot-mqtt-bridge", "iot-kafka-consumer"} {
for _, appLabel := range []string{"iot-mqtt-bridge", "iot-sqs-consumer"} {
podList := &corev1.PodList{}
if err := h.K8s.List(ctx, podList,
client.InNamespace("sless"),