shared-sqs: Этап 1 — клон GoAWS, удаление SNS, go build OK

shared-sqs/app/conf/config.go

@@ -0,0 +1,194 @@
+package conf
+
+import (
+"encoding/json"
+"fmt"
+"io/fs"
+"os"
+"path/filepath"
+"strconv"
+"strings"
+"time"
+
+"shared-sqs/app/models"
+"shared-sqs/app/utils"
+
+log "github.com/sirupsen/logrus"
+
+"github.com/ghodss/yaml"
+)
+
+var envs map[string]models.Environment
+
+func LoadYamlConfig(filename string, env string) []string {
+ports := []string{"4100"}
+
+if filename == "" {
+root, _ := filepath.Abs(".")
+err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+if "goaws.yaml" == d.Name() {
+filename = path
+}
+return nil
+})
+if err != nil || filename == "" {
+log.Warn("Failure to find default config file")
+return ports
+}
+}
+
+filename, _ = filepath.Abs(filename)
+if _, err := os.Stat(filename); err != nil {
+log.Warnf("Failure to find config file: %s", filename)
+return ports
+}
+
+log.Infof("Loading config file: %s", filename)
+yamlFile, err := os.ReadFile(filename)
+if err != nil {
+return ports
+}
+
+err = yaml.Unmarshal(yamlFile, &envs)
+if err != nil {
+log.Errorf("err: %v\n", err)
+return ports
+}
+if env == "" {
+env = "Local"
+}
+
+if envs[env].Region == "" {
+models.CurrentEnvironment.Region = "local"
+}
+
+models.CurrentEnvironment = envs[env]
+
+if envs[env].Port != "" {
+ports = []string{envs[env].Port}
+}
+
+models.LogMessages = false
+models.LogFile = "./goaws_messages.log"
+if envs[env].LogToFile == true {
+models.LogMessages = true
+if envs[env].LogFile != "" {
+models.LogFile = envs[env].LogFile
+}
+}
+
+if models.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout <= 0 {
+models.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout = 30
+}
+
+if models.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize <= 0 {
+models.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize = 262144 // 256K
+}
+
+if models.CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod <= 0 {
+models.CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod = 345600 // 4 days
+}
+
+if models.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds <= 0 {
+models.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds = 0
+}
+
+if models.CurrentEnvironment.AccountID == "" {
+models.CurrentEnvironment.AccountID = "queue"
+}
+
+if models.CurrentEnvironment.Host == "" {
+models.CurrentEnvironment.Host = "localhost"
+models.CurrentEnvironment.Port = "4100"
+}
+
+models.SyncQueues.Lock()
+for _, queue := range envs[env].Queues {
+queueUrl := "http://" + models.CurrentEnvironment.Host + ":" + models.CurrentEnvironment.Port +
+"/" + models.CurrentEnvironment.AccountID + "/" + queue.Name
+if models.CurrentEnvironment.Region != "" {
+queueUrl = "http://" + models.CurrentEnvironment.Region + "." + models.CurrentEnvironment.Host + ":" +
+models.CurrentEnvironment.Port + "/" + models.CurrentEnvironment.AccountID + "/" + queue.Name
+}
+queueArn := "arn:aws:sqs:" + models.CurrentEnvironment.Region + ":" + models.CurrentEnvironment.AccountID + ":" + queue.Name
+
+if queue.ReceiveMessageWaitTimeSeconds == 0 {
+queue.ReceiveMessageWaitTimeSeconds = models.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds
+}
+if queue.MaximumMessageSize == 0 {
+queue.MaximumMessageSize = models.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize
+}
+if queue.VisibilityTimeout == 0 {
+queue.VisibilityTimeout = models.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout
+}
+if queue.MessageRetentionPeriod == 0 {
+queue.MessageRetentionPeriod = models.CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod
+}
+
+models.SyncQueues.Queues[queue.Name] = &models.Queue{
+Name:                          queue.Name,
+VisibilityTimeout:             queue.VisibilityTimeout,
+Arn:                           queueArn,
+URL:                           queueUrl,
+ReceiveMessageWaitTimeSeconds: queue.ReceiveMessageWaitTimeSeconds,
+MaximumMessageSize:            queue.MaximumMessageSize,
+MessageRetentionPeriod:        queue.MessageRetentionPeriod,
+IsFIFO:                        utils.HasFIFOQueueName(queue.Name),
+EnableDuplicates:              models.CurrentEnvironment.EnableDuplicates,
+Duplicates:                    make(map[string]time.Time),
+}
+}
+
+// Второй проход — устанавливаем RedrivePolicy, чтобы DLQ были доступны независимо от порядка
+for _, queue := range envs[env].Queues {
+q := models.SyncQueues.Queues[queue.Name]
+if queue.RedrivePolicy != "" {
+err := setQueueRedrivePolicy(models.SyncQueues.Queues, q, queue.RedrivePolicy)
+if err != nil {
+log.Errorf("err: %s", err)
+return ports
+}
+}
+}
+
+models.SyncQueues.Unlock()
+
+return ports
+}
+
+func setQueueRedrivePolicy(queues map[string]*models.Queue, q *models.Queue, strRedrivePolicy string) error {
+// Поддерживаем maxReceiveCount как int и как string (AWS SDK использует string)
+redrivePolicy1 := struct {
+MaxReceiveCount     int    `json:"maxReceiveCount"`
+DeadLetterTargetArn string `json:"deadLetterTargetArn"`
+}{}
+redrivePolicy2 := struct {
+MaxReceiveCount     string `json:"maxReceiveCount"`
+DeadLetterTargetArn string `json:"deadLetterTargetArn"`
+}{}
+err1 := json.Unmarshal([]byte(strRedrivePolicy), &redrivePolicy1)
+err2 := json.Unmarshal([]byte(strRedrivePolicy), &redrivePolicy2)
+maxReceiveCount := redrivePolicy1.MaxReceiveCount
+deadLetterQueueArn := redrivePolicy1.DeadLetterTargetArn
+if err1 != nil && err2 != nil {
+return fmt.Errorf("invalid json for queue redrive policy ")
+} else if err1 != nil {
+maxReceiveCount, _ = strconv.Atoi(redrivePolicy2.MaxReceiveCount)
+deadLetterQueueArn = redrivePolicy2.DeadLetterTargetArn
+}
+
+if (deadLetterQueueArn != "" && maxReceiveCount == 0) ||
+(deadLetterQueueArn == "" && maxReceiveCount != 0) {
+return fmt.Errorf("invalid redrive policy values")
+}
+dlt := strings.Split(deadLetterQueueArn, ":")
+deadLetterQueueName := dlt[len(dlt)-1]
+deadLetterQueue, ok := queues[deadLetterQueueName]
+if !ok {
+return fmt.Errorf("deadletter queue not found")
+}
+q.DeadLetterQueue = deadLetterQueue
+q.MaxReceiveCount = maxReceiveCount
+
+return nil
+}

shared-sqs/app/conf/config_test.go

@@ -0,0 +1,166 @@
+package conf
+
+import (
+	"testing"
+
+	"shared-sqs/app/models"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfig_NoQueuesOrTopics(t *testing.T) {
+	env := "NoQueuesOrTopics"
+	port := LoadYamlConfig("./mock-data/mock-config.yaml", env)
+	if port[0] != "4100" {
+		t.Errorf("Expected port number 4200 but got %s\n", port)
+	}
+
+	numQueues := len(envs[env].Queues)
+	if numQueues != 0 {
+		t.Errorf("Expected zero queues to be in the environment but got %d\n", numQueues)
+	}
+	numQueues = len(models.SyncQueues.Queues)
+	if numQueues != 0 {
+		t.Errorf("Expected zero queues to be in the sqs topics but got %d\n", numQueues)
+	}
+
+	numTopics := len(envs[env].Topics)
+	if numTopics != 0 {
+		t.Errorf("Expected zero topics to be in the environment but got %d\n", numTopics)
+	}
+	numTopics = len(models.SyncTopics.Topics)
+	if numTopics != 0 {
+		t.Errorf("Expected zero topics to be in the sns topics but got %d\n", numTopics)
+	}
+}
+
+func TestConfig_CreateQueuesTopicsAndSubscriptions(t *testing.T) {
+	env := "Local"
+	port := LoadYamlConfig("./mock-data/mock-config.yaml", env)
+	if port[0] != "4100" {
+		t.Errorf("Expected port number 4100 but got %s\n", port)
+	}
+
+	numQueues := len(envs[env].Queues)
+	if numQueues != 4 {
+		t.Errorf("Expected three queues to be in the environment but got %d\n", numQueues)
+	}
+	numQueues = len(models.SyncQueues.Queues)
+	if numQueues != 6 {
+		t.Errorf("Expected five queues to be in the sqs topics but got %d\n", numQueues)
+	}
+
+	numTopics := len(envs[env].Topics)
+	if numTopics != 2 {
+		t.Errorf("Expected two topics to be in the environment but got %d\n", numTopics)
+	}
+	numTopics = len(models.SyncTopics.Topics)
+	if numTopics != 2 {
+		t.Errorf("Expected two topics to be in the sns topics but got %d\n", numTopics)
+	}
+}
+
+func TestConfig_QueueAttributes(t *testing.T) {
+	var emptyQueue *models.Queue
+	env := "Local"
+	port := LoadYamlConfig("./mock-data/mock-config.yaml", env)
+	if port[0] != "4100" {
+		t.Errorf("Expected port number 4100 but got %s\n", port)
+	}
+
+	assert.Equal(t, 10, models.SyncQueues.Queues["local-queue1"].ReceiveMessageWaitTimeSeconds)
+	assert.Equal(t, 10, models.SyncQueues.Queues["local-queue1"].VisibilityTimeout)
+	assert.Equal(t, 1024, models.SyncQueues.Queues["local-queue1"].MaximumMessageSize)
+	assert.Equal(t, emptyQueue, models.SyncQueues.Queues["local-queue1"].DeadLetterQueue)
+	assert.Equal(t, 0, models.SyncQueues.Queues["local-queue1"].MaxReceiveCount)
+	assert.Equal(t, 345600, models.SyncQueues.Queues["local-queue1"].MessageRetentionPeriod)
+	assert.Equal(t, 100, models.SyncQueues.Queues["local-queue3"].MaxReceiveCount)
+
+	assert.Equal(t, "local-queue3-dlq", models.SyncQueues.Queues["local-queue3"].DeadLetterQueue.Name)
+	assert.Equal(t, 128, models.SyncQueues.Queues["local-queue2"].MaximumMessageSize)
+	assert.Equal(t, 150, models.SyncQueues.Queues["local-queue2"].VisibilityTimeout)
+	assert.Equal(t, 245600, models.SyncQueues.Queues["local-queue2"].MessageRetentionPeriod)
+}
+
+func TestConfig_NoQueueAttributeDefaults(t *testing.T) {
+	env := "NoQueueAttributeDefaults"
+	LoadYamlConfig("./mock-data/mock-config.yaml", env)
+
+	receiveWaitTime := models.SyncQueues.Queues["local-queue1"].ReceiveMessageWaitTimeSeconds
+	if receiveWaitTime != 0 {
+		t.Errorf("Expected local-queue1 Queue to be configured with ReceiveMessageWaitTimeSeconds: 0 but got %d\n", receiveWaitTime)
+	}
+	timeoutSecs := models.SyncQueues.Queues["local-queue1"].VisibilityTimeout
+	if timeoutSecs != 30 {
+		t.Errorf("Expected local-queue1 Queue to be configured with VisibilityTimeout: 30 but got %d\n", timeoutSecs)
+	}
+
+	receiveWaitTime = models.SyncQueues.Queues["local-queue2"].ReceiveMessageWaitTimeSeconds
+	if receiveWaitTime != 20 {
+		t.Errorf("Expected local-queue2 Queue to be configured with ReceiveMessageWaitTimeSeconds: 20 but got %d\n", receiveWaitTime)
+	}
+
+	messageRetentionPeriod := models.SyncQueues.Queues["local-queue1"].MessageRetentionPeriod
+	if messageRetentionPeriod != 345600 {
+		t.Errorf("Expected local-queue2 Queue to be configured with VisibilityTimeout: 150 but got %d\n", timeoutSecs)
+	}
+}
+
+func TestConfig_invalid_config_resorts_to_default_queue_attributes(t *testing.T) {
+	env := "missing"
+	port := LoadYamlConfig("./mock-data/mock-config.yaml", env)
+	if port[0] != "4100" {
+		t.Errorf("Expected port number 4100 but got %s\n", port)
+	}
+
+	assert.Equal(t, 262144, models.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize)
+	assert.Equal(t, 345600, models.CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod)
+	assert.Equal(t, 0, models.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds)
+	assert.Equal(t, 30, models.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout)
+}
+
+func TestConfig_LoadYamlConfig_finds_default_config(t *testing.T) {
+	expectedQueues := []string{
+		"local-queue1",
+		"local-queue2",
+		"local-queue3",
+		"local-queue3-dlq",
+		"local-queue4",
+	}
+	expectedTopics := []string{
+		"local-topic1",
+		"local-topic2",
+		"local-topic3",
+		"local-topic4",
+	}
+
+	env := "Local"
+	LoadYamlConfig("", env)
+
+	queues := models.SyncQueues.Queues
+	topics := models.SyncTopics.Topics
+	for _, expectedName := range expectedQueues {
+		_, ok := queues[expectedName]
+		assert.True(t, ok)
+	}
+	for _, expectedName := range expectedTopics {
+		_, ok := topics[expectedName]
+		assert.True(t, ok)
+	}
+}
+
+func TestConfig_LoadYamlConfig_missing_config_loads_nothing(t *testing.T) {
+	models.CurrentEnvironment = models.Environment{}
+	ports := LoadYamlConfig("/garbage", "Local")
+
+	assert.Equal(t, []string{"4100"}, ports)
+	assert.Equal(t, models.CurrentEnvironment, models.Environment{})
+}
+
+func TestConfig_LoadYamlConfig_invalid_config_loads_nothing(t *testing.T) {
+	models.CurrentEnvironment = models.Environment{}
+	ports := LoadYamlConfig("../common/common.go", "Local")
+
+	assert.Equal(t, []string{"4100"}, ports)
+	assert.Equal(t, models.CurrentEnvironment, models.Environment{})
+}

shared-sqs/app/conf/goaws.yaml

@@ -0,0 +1,66 @@
+Local:                              # Environment name that can be passed on the command line
+  #     (i.e.: ./goaws [Local | Dev]  -- defaults to 'Local')
+  Host: goaws.com                   # hostname of the goaws system  (for docker-compose this is the tag name of the container)
+  # you can now use either 1 port for both sns and sqs or alternatively you can comment out Port and use SqsPort + SnsPort for compatibilyt with
+  # yopa and (fage-sns + face-sqs).  If both ways are in the config file on the one "Port" will be used by GoAws
+  Port: 4100                        # port to listen on.
+  # SqsPort: 9324                     # alterante Sqs Port
+  # SnsPort: 9292                     # alternate Sns Port
+  Region: us-east-1
+  AccountId: "100010001000"
+  LogToFile: false                 # Log messages (true/false)
+  LogFile: .st/goaws_messages.log  # Log filename (for message logging
+  EnableDuplicates: false           # Enable or not deduplication based on messageDeduplicationId
+  QueueAttributeDefaults:           # default attributes for all queues
+    VisibilityTimeout: 30              # message visibility timeout
+    ReceiveMessageWaitTimeSeconds: 0   # receive message max wait time
+    MaximumMessageSize: 262144         # maximum message size (bytes)
+#    MessageRetentionPeriod: 445600     # time period to retain messages (seconds) NOTE: Functionality not implemented
+  Queues:                           # List of queues to create at startup
+    - Name: local-queue1                # Queue name
+    - Name: local-queue2                # Queue name
+      ReceiveMessageWaitTimeSeconds: 20 # Queue receive message max wait time
+    - Name: local-queue3                # Queue name
+      RedrivePolicy: '{"maxReceiveCount": 100, "deadLetterTargetArn":"arn:aws:sqs:us-east-1:100010001000:local-queue3-dlq"}'
+    - Name: local-queue3-dlq            # Queue name
+  Topics:                           # List of topic to create at startup
+    - Name: local-topic1            # Topic name - with some Subscriptions
+      Subscriptions:                # List of Subscriptions to create for this topic (queues will be created as required)
+        - QueueName: local-queue3   # Queue name
+          Raw: false                # Raw message delivery (true/false)
+        - QueueName: local-queue4   # Queue name
+          Raw: true                 # Raw message delivery (true/false)
+          #FilterPolicy: '{"foo": ["bar"]}' # Subscription's FilterPolicy, json object as a string
+    - Name: local-topic2            # Topic name - no Subscriptions
+    - Name: local-topic3            # Topic name - http subscription
+      Subscriptions:
+        - Protocol: https
+          EndPoint: https://enkrogwitfcgi.x.pipedream.net
+          TopicArn: arn:aws:sns:us-east-1:100010001000:local-topic2
+          FilterPolicy: '{"event": ["my_event"]}'
+          Raw: true
+    - Name: local-topic4
+  RandomLatency:                    # Parameters for introducing random latency into message queuing
+    Min: 0                          # Desired latency in milliseconds, if min and max are zero, no latency will be applied.
+    Max: 0                          # Desired latency in milliseconds
+
+Dev:                                # Another environment
+  Host: localhost
+  Port: 4100
+  # SqsPort: 9324
+  # SnsPort: 9292
+  AccountId: "794373491471"
+  LogToFile: false
+  LogFile: ./goaws_messages.log
+  Queues:
+    - Name: dev-queue1
+    - Name: dev-queue2
+  Topics:
+    - Name: dev-topic1
+      Subscriptions:
+        - QueueName: dev-queue3
+          Raw: false
+        - QueueName: dev-queue4
+          Raw: true
+    - Name: dev-topic2
+

shared-sqs/app/conf/mock-data/mock-config.yaml

@@ -0,0 +1,79 @@
+Local:
+  Host: localhost
+  Port: 4100
+  Region: us-east-1
+  AccountId: "100010001000"
+  LogMessages: true
+  LogFile: ./goaws_messages.log
+  QueueAttributeDefaults:
+    VisibilityTimeout: 10
+    ReceiveMessageWaitTimeSeconds: 10
+    MaximumMessageSize: 1024
+  Queues:
+    - Name: local-queue1
+    - Name: local-queue2
+      ReceiveMessageWaitTimeSeconds: 20
+      MaximumMessageSize: 128
+      VisibilityTimeout: 150
+      MessageRetentionPeriod: 245600
+    - Name: local-queue3
+      RedrivePolicy: '{"maxReceiveCount": 100, "deadLetterTargetArn":"arn:aws:sqs:us-east-1:100010001000:local-queue3-dlq"}'
+    - Name: local-queue3-dlq
+  Topics:
+    - Name: local-topic1
+      Subscriptions:
+        - QueueName: local-queue4
+          Raw: false
+        - QueueName: local-queue5
+          Raw: true
+          FilterPolicy: '{"foo":["bar"]}'
+    - Name: local-topic2
+
+NoQueuesOrTopics:
+  Host: localhost
+  Port: 4100
+  LogMessages: true
+  LogFile: ./goaws_messages.log
+  Region: eu-west-1
+
+NoQueueAttributeDefaults:
+  Host: localhost
+  Port: 4100
+  LogMessages: true
+  LogFile: ./goaws_messages.log
+  Region: eu-west-1
+  Queues:
+    - Name: local-queue1
+    - Name: local-queue2
+      ReceiveMessageWaitTimeSeconds: 20
+
+BaseUnitTests:
+  Host: host
+  Port: port
+  Region: region
+  AccountId: accountID
+  LogMessages: true
+  LogFile: ./goaws_messages.log
+  Queues:
+    - Name: unit-queue1
+    - Name: unit-queue2
+      RedrivePolicy: '{"maxReceiveCount": 1, "deadLetterTargetArn":"arn:aws:sqs:us-east-1:100010001000:dead-letter-queue1"}'
+    - Name: dead-letter-queue1
+    - Name: subscribed-queue1
+    - Name: subscribed-queue3
+  Topics:
+    - Name: unit-topic1
+      Subscriptions:
+        - QueueName: subscribed-queue1
+          Raw: true
+    - Name: unit-topic2
+    - Name: unit-topic3
+      Subscriptions:
+        - QueueName: subscribed-queue3
+          Raw: false
+    - Name: unit-topic-http
+      Subscriptions:
+        - Protocol: http
+          EndPoint: http://over.ride.me/for/tests
+          TopicArn: arn:aws:sqs:region:accountID:unit-topic-http
+          Raw: true