diff --git a/shared-sqs/.gitignore b/shared-sqs/.gitignore new file mode 100644 index 0000000..53fdbc5 --- /dev/null +++ b/shared-sqs/.gitignore @@ -0,0 +1,6 @@ +# Бинарник сервиса +shared-sqs + +# Go build cache +*.test +*.out diff --git a/shared-sqs/Dockerfile b/shared-sqs/Dockerfile new file mode 100644 index 0000000..7927fd8 --- /dev/null +++ b/shared-sqs/Dockerfile @@ -0,0 +1,28 @@ +# build image +FROM golang:alpine as build + +WORKDIR /go/src/github.com/Admiral-Piett/goaws + +COPY ./app/ ./app/ +COPY ./go.mod . +COPY ./go.sum . + +RUN ls -la +RUN CGO_ENABLED=0 go test ./app/... +RUN go build -o goaws app/cmd/goaws.go + +# release image +FROM alpine + +WORKDIR /app + +COPY --from=build /go/src/github.com/Admiral-Piett/goaws/goaws ./goaws + +COPY app/conf/goaws.yaml ./conf/ + +EXPOSE 4100 + +HEALTHCHECK --interval=5s --timeout=3s --retries=3 \ + CMD wget localhost:4100/health -q -O - > /dev/null + +ENTRYPOINT ["./goaws"] diff --git a/shared-sqs/PLAN.md b/shared-sqs/PLAN.md new file mode 100644 index 0000000..fdcb021 --- /dev/null +++ b/shared-sqs/PLAN.md @@ -0,0 +1,1693 @@ +# shared-sqs — План реализации + +**Дата:** 2026-04-09 +**Исполнитель:** Claude Sonnet (или другой агент) +**Подготовил:** Claude Opus 4 (анализ GoAWS, архитектура, ловушки) + +--- + +## 1. ЧТО ЭТО + +Multi-tenant SQS-совместимый сервис на базе форка [GoAWS](https://github.com/Admiral-Piett/goaws) (Go, MIT, 835 stars). + +**Отличие от sqs-operator:** sqs-operator деплоит каждому тенанту ОТДЕЛЬНЫЙ pod с ElasticMQ (~300MB RAM каждый). shared-sqs — ОДИН pod обслуживает ВСЕХ тенантов (~50MB RAM base). + +**Что shared-sqs делает:** +- SQS-совместимый API (CreateQueue, SendMessage, ReceiveMessage, DeleteMessage и т.д.) +- Аутентификация по AccessKeyId (из AWS Authorization header) +- Изоляция очередей между тенантами (тенант видит ТОЛЬКО свои очереди) +- Admin API для управления тенантами (CRUD) +- Работает с AWS CLI и AWS SDK без модификаций + +--- + +## 2. АРХИТЕКТУРА GoAWS (то, что форкаем) + +### 2.1 Структура исходников +``` +app/ +├── cmd/goaws.go # Entry point (~40 LOC): флаги, загрузка конфига, HTTP сервер +├── conf/ # Загрузка YAML конфига +├── gosqs/ # SQS handlers (ЯДРО — ~20 файлов) +│ ├── create_queue.go # CreateQueueV1() +│ ├── send_message.go # SendMessageV1() +│ ├── receive_message.go # ReceiveMessageV1() +│ ├── delete_message.go # DeleteMessageV1() +│ ├── delete_message_batch.go +│ ├── delete_queue.go +│ ├── get_queue_attributes.go +│ ├── get_queue_url.go +│ ├── list_queues.go +│ ├── purge_queue.go +│ ├── send_message_batch.go +│ ├── set_queue_attributes.go +│ ├── change_message_visibility.go +│ ├── queue_attributes.go # Helpers для атрибутов +│ └── gosqs.go # PeriodicTasks (visibility timeout, DLQ, dedup) +├── gosns/ # SNS handlers — НЕ НУЖНЫ, УДАЛИТЬ +├── models/ +│ ├── globals.go # SyncQueues, SyncTopics — глобальные map + RWMutex +│ ├── models.go # Queue, SqsMessage, Topic structs +│ ├── configuration.go # Environment, EnvQueue, config structs +│ ├── constants.go +│ ├── conversions.go # Парсинг тел запросов +│ ├── errors.go # AWS-совместимые ошибки +│ ├── helpers.go +│ ├── requests.go # Request structs (CreateQueueRequest, SendMessageRequest и т.д.) +│ └── responses.go # Response structs (XML + JSON) +├── router/ +│ └── router.go # gorilla/mux, actionHandler, routingTableV1 +├── interfaces/ # AbstractResponseBody interface +├── utils/ # Hash, MD5, REQUEST_TRANSFORMER +├── mocks/ # Тестовые моки +├── fixtures/ # Тестовые данные +├── servertest/ +└── test/ +``` + +### 2.2 Критические архитектурные точки + +**Глобальный state** (`models/globals.go`): +```go +var SyncQueues = struct { + sync.RWMutex + Queues map[string]*Queue +}{Queues: make(map[string]*Queue)} +``` +Все очереди храняться В ОДНОМ map. Ключ = имя очереди (string). + +**Роутинг** (`router/router.go`): +```go +r.HandleFunc("/", actionHandler) +r.HandleFunc("/{account}", actionHandler) +r.HandleFunc("/queue/{queueName}", actionHandler) +r.HandleFunc("/{account}/{queueName}", actionHandler) +``` +Все запросы идут в `actionHandler`, который извлекает `Action` из: +- Query param `Action=CreateQueue` (AWS Query Protocol) +- Header `X-Amz-Target: AmazonSQS.CreateQueue` (AWS JSON Protocol) + +**Dispatch table** (`router/router.go`): +```go +var routingTableV1 = map[string]func(r *http.Request) (int, interfaces.AbstractResponseBody){ + "CreateQueue": sqs.CreateQueueV1, + "SendMessage": sqs.SendMessageV1, + "ReceiveMessage": sqs.ReceiveMessageV1, + // ... все SQS + SNS actions +} +``` + +**URL конструкция** (в create_queue.go): +```go +queueUrl := "http://" + host + ":" + port + "/" + accountID + "/" + queueName +queueArn := "arn:aws:sqs:" + region + ":" + accountID + ":" + queueName +``` +`accountID` берётся из `models.CurrentEnvironment.AccountID` — ГЛОБАЛЬНАЯ переменная (одна на всех). + +**Зависимости** (go.mod): +- `gorilla/mux v1.8.0` — роутер +- `gorilla/schema v1.4.1` — form decoder +- `google/uuid v1.6.0` — UUID генерация +- `sirupsen/logrus` — логирование +- `ghodss/yaml` — YAML парсинг +- `aws/aws-sdk-go v1.47.3` — только для тестов + +--- + +## 3. ПЛАН ИЗМЕНЕНИЙ + +### 3.0 Общие правила работы + +**КРИТИЧНО — все команды ТОЛЬКО через SSH:** +``` +ssh -i /home/naeel/.ssh/naeel_vm_id_ed25519 -o StrictHostKeyChecking=no naeel@5.172.178.213 КОМАНДА +``` + +**Go файлы редактировать ТОЛЬКО через Python patch скрипты на VM**, НЕ через sshfs/VS Code. + +**Markdown/yaml/conf файлы** можно создавать через `cat > file << EOF` по SSH. + +**Перед go build** — убедиться что бинарник в `.gitignore`. + +### Этап 1: Клон GoAWS + чистка (1 час) + +**Действия:** + +1. Клонировать GoAWS в `shared-sqs/`: +```bash +cd ~/terra/sless/shared-sqs +git clone https://github.com/Admiral-Piett/goaws.git _upstream +# Копируем ТОЛЬКО нужное: +cp -r _upstream/app ./app +cp _upstream/go.mod ./go.mod +cp _upstream/go.sum ./go.sum +cp _upstream/Dockerfile ./Dockerfile +rm -rf _upstream +``` + +2. Поменять module name в go.mod: +``` +module shared-sqs +go 1.22 +``` +(Повысить версию Go с 1.18 до 1.22+) + +3. Обновить все import paths: + - Заменить `github.com/Admiral-Piett/goaws/app/` → `shared-sqs/app/` + - Это во ВСЕХ .go файлах + +4. УДАЛИТЬ всё связанное с SNS: + - `app/gosns/` — целиком + - Из `router/router.go` — убрать все SNS записи из `routingTableV1` + - Из `models/globals.go` — убрать `SyncTopics` + - Из `models/models.go` — убрать `Topic`, `Subscription`, `SNSMessage`, `FilterPolicy` + - Из `models/configuration.go` — убрать `EnvTopic`, `EnvSubsciption` + - Из `models/requests.go` и `responses.go` — убрать SNS-related structs + +5. УДАЛИТЬ тестовые/mock директории (мы напишем свои тесты): + - `app/mocks/` + - `app/fixtures/` + - `app/servertest/` + - `app/test/` + - `app/smoke_tests/` (если скопировалась) + +6. Проверить что компилируется: +```bash +cd ~/terra/sless/shared-sqs +go mod tidy +go build -o shared-sqs app/cmd/goaws.go +``` + +7. Проверить что стартует: +```bash +./shared-sqs -debug +# В другом окне: curl http://localhost:4100/health +# Ожидание: 200 OK +``` + +**Тест прохождения этапа:** `go build` успешен, `/health` возвращает 200. + +--- + +### Этап 2: Tenant Model + хранилище (30 мин) + +**Создать файл `app/tenant/tenant.go`:** + +```go +package tenant + +import ( + "crypto/rand" + "encoding/hex" + "sync" + "time" +) + +// Tenant — модель тенанта shared-sqs +type Tenant struct { + ID string // уникальный идентификатор (генерируется) + Name string // имя тенанта (для отображения) + AccessKey string // аналог AWS AccessKeyId + SecretKey string // аналог AWS SecretAccessKey + MaxQueues int // лимит очередей (0 = безлимит) + CreatedAt time.Time + Active bool +} + +// TenantStore — in-memory хранилище тенантов +type TenantStore struct { + sync.RWMutex + // byID — поиск по ID + byID map[string]*Tenant + // byAccessKey — поиск по AccessKey (для auth) + byAccessKey map[string]*Tenant +} + +// NewTenantStore — создаёт пустое хранилище +func NewTenantStore() *TenantStore { ... } + +// Create — создаёт нового тенанта, генерирует ключи +func (s *TenantStore) Create(name string, maxQueues int) (*Tenant, error) { ... } + +// GetByAccessKey — поиск тенанта по AccessKeyId (для auth middleware) +func (s *TenantStore) GetByAccessKey(accessKey string) (*Tenant, bool) { ... } + +// GetByID — поиск по ID +func (s *TenantStore) GetByID(id string) (*Tenant, bool) { ... } + +// Delete — удаление тенанта +func (s *TenantStore) Delete(id string) bool { ... } + +// List — список всех тенантов +func (s *TenantStore) List() []*Tenant { ... } +``` + +**Генерация ключей (БЕЗОПАСНАЯ):** +```go +func generateAccessKey() string { + // Формат: SSAK-{random hex 12} (SS = Shared SQS) + b := make([]byte, 12) + rand.Read(b) + return "SSAK-" + hex.EncodeToString(b) +} + +func generateSecretKey() string { + // 32 байта random → 64 hex символа + b := make([]byte, 32) + rand.Read(b) + return hex.EncodeToString(b) +} +``` + +**ЛОВУШКА #1:** Используй `crypto/rand`, НЕ `math/rand` для ключей. `math/rand` предсказуем. + +**ЛОВУШКА #2:** Два индекса (byID и byAccessKey) — при Delete надо удалить из ОБОИХ. + +**Тест прохождения:** TenantStore создаёт/ищет/удаляет тенантов. Ключи уникальны. + +--- + +### Этап 3: Auth Middleware (30 мин) + +**Создать файл `app/auth/middleware.go`:** + +```go +package auth + +import ( + "context" + "net/http" + "strings" +) + +type contextKey string +const TenantContextKey contextKey = "tenant" + +// AuthMiddleware — извлекает AccessKeyId из AWS Authorization header +// и находит тенанта в store +func AuthMiddleware(store *tenant.TenantStore) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // /health и /admin/** — пропускаем (разная auth) + if r.URL.Path == "/health" { + next.ServeHTTP(w, r) + return + } + if strings.HasPrefix(r.URL.Path, "/admin/") { + next.ServeHTTP(w, r) + return + } + + accessKeyId := extractAccessKeyId(r) + if accessKeyId == "" { + writeAuthError(w, "MissingAuthenticationToken") + return + } + + t, ok := store.GetByAccessKey(accessKeyId) + if !ok || !t.Active { + writeAuthError(w, "InvalidClientTokenId") + return + } + + ctx := context.WithValue(r.Context(), TenantContextKey, t) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} +``` + +**Извлечение AccessKeyId из Authorization header:** +```go +// extractAccessKeyId — извлекает AWS AccessKeyId из запроса +// Формат header: "AWS4-HMAC-SHA256 Credential={AccessKeyId}/{date}/{region}/sqs/aws4_request, ..." +func extractAccessKeyId(r *http.Request) string { + // Вариант 1: Authorization header (AWS Signature V4) + auth := r.Header.Get("Authorization") + if strings.HasPrefix(auth, "AWS4-HMAC-SHA256") { + // Ищем "Credential=" и берём всё до первого "/" + idx := strings.Index(auth, "Credential=") + if idx >= 0 { + rest := auth[idx+len("Credential="):] + slashIdx := strings.Index(rest, "/") + if slashIdx > 0 { + return rest[:slashIdx] + } + } + } + + // Вариант 2: Query parameter (presigned URLs) + if cred := r.URL.Query().Get("X-Amz-Credential"); cred != "" { + parts := strings.SplitN(cred, "/", 2) + if len(parts) > 0 { + return parts[0] + } + } + + return "" +} +``` + +**ЛОВУШКА #3:** AWS CLI ВСЕГДА отправляет Authorization header с Signature V4. Наш middleware должен УМЕТЬ парсить его, даже если мы НЕ ПРОВЕРЯЕМ подпись. + +**ЛОВУШКА #4:** Когда AWS SDK делает `ReceiveMessage` с long polling (WaitTimeSeconds > 0), соединение держится до 20 секунд. Auth middleware не должен ставить таймаут короче. + +**ЛОВУШКА #5:** AWS SDK может отправлять запросы с `X-Amz-Security-Token` (STS). Игнорировать этот header. + +**Тест прохождения:** Запрос с `Authorization: AWS4-HMAC-SHA256 Credential=SSAK-abc123.../...` → тенант найден в контексте. Запрос без header → 403. + +--- + +### Этап 4: Queue Isolation — САМЫЙ КРИТИЧНЫЙ (2 часа) + +Это ЯДРО всех изменений. Все SQS handlers должны работать с tenant-scoped очередями. + +**4.1 Изменение ключей в SyncQueues** + +Сейчас: `SyncQueues.Queues["my-queue"]` +Станет: `SyncQueues.Queues["SSAK-abc123:my-queue"]` + +Формат внутреннего ключа: `{tenantAccessKey}:{queueName}` + +Почему AccessKey а не TenantID: AccessKey уже есть в auth context, не надо лишний lookup. AccessKey уникален. + +**4.2 Helper функции (создать `app/gosqs/tenant_helpers.go`):** + +```go +// tenantQueueKey — внутренний ключ очереди в SyncQueues +func tenantQueueKey(tenantAccessKey, queueName string) string { + return tenantAccessKey + ":" + queueName +} + +// getTenantFromContext — извлекает тенанта из request context +func getTenantFromContext(r *http.Request) *tenant.Tenant { + t, _ := r.Context().Value(auth.TenantContextKey).(*tenant.Tenant) + return t +} + +// tenantQueueUrl — формирует URL очереди для тенанта +func tenantQueueUrl(t *tenant.Tenant, queueName string) string { + return "http://" + models.CurrentEnvironment.Host + ":" + + models.CurrentEnvironment.Port + "/" + t.ID + "/" + queueName +} + +// tenantQueueArn — формирует ARN очереди +func tenantQueueArn(t *tenant.Tenant, queueName string) string { + return "arn:aws:sqs:" + models.CurrentEnvironment.Region + ":" + t.ID + ":" + queueName +} +``` + +**4.3 Изменения в КАЖДОМ handler (подробно)** + +**create_queue.go — CreateQueueV1:** +``` +БЫЛО: + queueName := requestBody.QueueName + key := queueName + url := http://host:port/accountID/queueName + +СТАЛО: + t := getTenantFromContext(req) + queueName := requestBody.QueueName + key := tenantQueueKey(t.AccessKey, queueName) + url := tenantQueueUrl(t, queueName) + arn := tenantQueueArn(t, queueName) + // Проверка лимита очередей: + if t.MaxQueues > 0 { + count := countTenantQueues(t.AccessKey) + if count >= t.MaxQueues { + return error "LimitExceeded" + } + } + models.SyncQueues.Queues[key] = queue +``` + +**send_message.go — SendMessageV1:** +``` +БЫЛО: + queueName = lastSegmentOfUrl(queueUrl) + _, ok := models.SyncQueues.Queues[queueName] + +СТАЛО: + t := getTenantFromContext(req) + queueName = lastSegmentOfUrl(queueUrl) + key := tenantQueueKey(t.AccessKey, queueName) + _, ok := models.SyncQueues.Queues[key] +``` + +**receive_message.go — ReceiveMessageV1:** — аналогично send_message + +**delete_message.go — DeleteMessageV1:** — аналогично + +**delete_message_batch.go — DeleteMessageBatchV1:** — аналогично + +**delete_queue.go — DeleteQueueV1:** +``` +СТАЛО: + t := getTenantFromContext(req) + key := tenantQueueKey(t.AccessKey, queueName) + delete(models.SyncQueues.Queues, key) +``` + +**list_queues.go — ListQueuesV1:** +``` +БЫЛО: + for _, queue := range models.SyncQueues.Queues { + urls = append(urls, queue.URL) + } + +СТАЛО: + t := getTenantFromContext(req) + prefix := t.AccessKey + ":" + for key, queue := range models.SyncQueues.Queues { + if strings.HasPrefix(key, prefix) { + urls = append(urls, queue.URL) + } + } +``` + +**get_queue_url.go — GetQueueUrlV1:** — добавить tenant prefix к lookup + +**get_queue_attributes.go — GetQueueAttributesV1:** — добавить tenant prefix + +**set_queue_attributes.go — SetQueueAttributesV1:** — добавить tenant prefix + +**purge_queue.go — PurgeQueueV1:** — добавить tenant prefix + +**change_message_visibility.go — ChangeMessageVisibilityV1:** — добавить tenant prefix + +**send_message_batch.go — SendMessageBatchV1:** — добавить tenant prefix + +**gosqs.go — PeriodicTasks:** — НЕ ТРОГАТЬ. Он итерирует ВСЕ очереди по полному ключу, не по имени. Работает корректно. + +**ЛОВУШКА #6:** `SendMessageV1` извлекает queueName из QueueUrl через `strings.Split(url, "/")` и берёт ПОСЛЕДНИЙ сегмент. Если URL = `http://host:port/tenantID/myqueue`, последний сегмент = `myqueue` — это ПРАВИЛЬНО, не ломается. + +**ЛОВУШКА #7:** `getQueueFromPath()` в gosqs.go тоже парсит URL. Убедиться что при `/{tenantID}/{queueName}` парсинг берёт queueName, а не tenantID. + +**ЛОВУШКА #8:** FIFO очереди имеют имена вида `myqueue.fifo`. Ключ будет `SSAK-xxx:myqueue.fifo` — это OK, `.fifo` стоит в конце имени, не ключа. Проверить что `utils.HasFIFOQueueName()` получает `queueName`, а не `key`. + +**ЛОВУШКА #9:** RedrivePolicy содержит ARN target очереди. При парсинге ARN в DLQ setup — извлечь имя очереди из ARN, затем добавить tenant prefix для lookup. ОБА (основная и DLQ) должны принадлежать одному тенанту. + +**ЛОВУШКА #10:** `QueueUrl` в ответах CreateQueue и GetQueueUrl используется AWS SDK для всех последующих вызовов. Если формат URL неправильный — SDK не сможет Send/Receive. URL ОБЯЗАН содержать tenantID в пути: `http://host:port/{tenantID}/{queueName}`. + +**4.4 Изменения в роутере** + +В `router.go` — маршрут `/{account}/{queueName}` уже существует. `{account}` = наш `{tenantID}`. Но нужно добавить middleware: + +```go +func New(tenantStore *tenant.TenantStore) http.Handler { + r := mux.NewRouter() + r.HandleFunc("/health", health).Methods("GET") + + // Admin API — отдельная auth (bearer token) + admin := r.PathPrefix("/admin").Subrouter() + // ... admin routes (см. Этап 5) + + // SQS API — tenant auth + sqsRouter := r.PathPrefix("/").Subrouter() + sqsRouter.Use(auth.AuthMiddleware(tenantStore)) + sqsRouter.HandleFunc("/", actionHandler).Methods("GET", "POST") + sqsRouter.HandleFunc("/{account}", actionHandler).Methods("GET", "POST") + sqsRouter.HandleFunc("/queue/{queueName}", actionHandler).Methods("GET", "POST") + sqsRouter.HandleFunc("/{account}/{queueName}", actionHandler).Methods("GET", "POST") + + return r +} +``` + +**Тест прохождения этапа 4:** +```bash +# Создать тенанта через Admin API +curl -X POST http://localhost:4100/admin/tenants -H "Authorization: Bearer $ADMIN_TOKEN" \ + -d name:test-tenant +# → возвращает access_key, secret_key + +# Создать очередь как тенант +aws --endpoint-url http://localhost:4100 sqs create-queue --queue-name test-q +# Настроить AWS credentials = access_key/secret_key из шага выше +# → возвращает QueueUrl с tenantID + +# Отправить и получить сообщение +aws --endpoint-url http://localhost:4100 sqs send-message --queue-url $QUEUE_URL --message-body "hello" +aws --endpoint-url http://localhost:4100 sqs receive-message --queue-url $QUEUE_URL +# → сообщение получено + +# Второй тенант НЕ видит очереди первого +# Со вторым access_key: +aws --endpoint-url http://localhost:4100 sqs list-queues +# → пустой список +``` + +--- + +### Этап 5: Admin API (30 мин) + +**Создать файл `app/admin/admin.go`:** + +Эндпоинты: +``` +POST /admin/tenants — создать тенанта +GET /admin/tenants — список тенантов +GET /admin/tenants/{id} — детали тенанта +DELETE /admin/tenants/{id} — удалить тенанта (и ВСЕ его очереди!) +GET /admin/health — детальный health (кол-во тенантов, очередей, сообщений) +``` + +**Auth для Admin API:** Header `Authorization: Bearer {admin_token}`. Admin token задаётся через: +- Переменная окружения `SHARED_SQS_ADMIN_TOKEN` +- Или в конфиге YAML + +**ЛОВУШКА #11:** При DELETE тенанта — ОБЯЗАТЕЛЬНО удалить ВСЕ его очереди из SyncQueues. Иначе memory leak. Итерировать SyncQueues.Queues, удалить все ключи с prefix `{accessKey}:`. + +**ЛОВУШКА #12:** Admin API НЕ должен быть доступен через тот же ingress что SQS API. Либо другой path prefix, либо другой порт. Рекомендация: path prefix `/admin/`, защищённый bearer token. НЕ ЗАБЫТЬ в auth middleware пропускать `/admin/` пути. + +**Request/Response форматы:** + +POST /admin/tenants: +```json +Request: {"name": "customer-001", "max_queues": 50} +Response: {"id": "t-a1b2c3", "name": "customer-001", "access_key": "SSAK-...", "secret_key": "...", "max_queues": 50} +``` +Важно: secret_key показывается ТОЛЬКО при создании. В List/Get — не включать. + +--- + +### Этап 6: Entry Point + Configuration (20 мин) + +**Модифицировать `app/cmd/goaws.go` (переименовать в `app/cmd/main.go`):** + +```go +func main() { + // Флаги + var configFile string + var adminToken string + var port string + flag.StringVar(&configFile, "config", "", "config file") + flag.StringVar(&adminToken, "admin-token", "", "admin API token") + flag.StringVar(&port, "port", "4100", "listen port") + flag.Parse() + + // Admin token: flag > env > config + if adminToken == "" { + adminToken = os.Getenv("SHARED_SQS_ADMIN_TOKEN") + } + if adminToken == "" { + log.Fatal("admin token required: use --admin-token or SHARED_SQS_ADMIN_TOKEN env") + } + + // Инициализация + tenantStore := tenant.NewTenantStore() + + // Загрузка конфига (если указан) — может содержать pre-created тенантов + if configFile != "" { + conf.LoadConfig(configFile, tenantStore) + } + + // Роутер + r := router.New(tenantStore, adminToken) + + // Periodic tasks + quit := make(chan bool) + go gosqs.PeriodicTasks(1*time.Second, quit) + + // Graceful shutdown + // ... (signal handling, quit channel) + + log.Infof("shared-sqs listening on 0.0.0.0:%s", port) + log.Fatal(http.ListenAndServe("0.0.0.0:"+port, r)) +} +``` + +**ЛОВУШКА #13:** GoAWS не обрабатывает сигналы. ДОБАВИТЬ graceful shutdown (SIGTERM/SIGINT → закрыть quit channel → остановить PeriodicTasks → дождаться завершения). + +--- + +### Этап 7: Dockerfile + Kubernetes (30 мин) + +**Dockerfile:** +```dockerfile +FROM golang:1.22-alpine AS builder +WORKDIR /build +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o shared-sqs app/cmd/main.go + +FROM alpine:3.19 +RUN apk --no-cache add ca-certificates +COPY --from=builder /build/shared-sqs /usr/local/bin/shared-sqs +EXPOSE 4100 +ENTRYPOINT ["shared-sqs"] +``` + +**Kubernetes manifests (`deployments/k8s/`):** + +deployment.yaml: +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shared-sqs + namespace: shared-sqs +spec: + replicas: 1 + strategy: + type: Recreate # НЕ RollingUpdate! Урок из ERR-SQS-06. + selector: + matchLabels: + app: shared-sqs + template: + spec: + containers: + - name: shared-sqs + image: pearlharbor.registryk8s.services.ngcloud.ru/naeel/shared-sqs:v0.1.0 + ports: + - containerPort: 4100 + env: + - name: SHARED_SQS_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: shared-sqs-admin + key: token + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 4100 + readinessProbe: + httpGet: + path: /health + port: 4100 +``` + +**ЛОВУШКА #14:** strategy: Recreate, НЕ RollingUpdate. In-memory state не шарится между подами. При RollingUpdate новый pod стартует с пустым state, а старый ещё жив = split brain. + +service.yaml, ingress.yaml — стандартные. + +**Реестр:** `pearlharbor.registryk8s.services.ngcloud.ru/naeel/shared-sqs` + +--- + +### Этап 8: Makefile (10 мин) + +```makefile +IMAGE_REPO=pearlharbor.registryk8s.services.ngcloud.ru/naeel/shared-sqs +VERSION=v0.1.0 + +build: +CGO_ENABLED=0 go build -o shared-sqs app/cmd/main.go + +docker-build: +docker build -t $(IMAGE_REPO):$(VERSION) . + +docker-push: +docker push $(IMAGE_REPO):$(VERSION) + +test: +go test ./... + +run: +./shared-sqs --admin-token=dev-token-123 --port=4100 -debug + +clean: +rm -f shared-sqs +``` + +--- + +### Этап 9: Тесты (1 час) + +**Создать `tests/shared_sqs_test.sh`** — bash тесты (как в customer-sqs): + +1. Admin API: + - Создать тенанта + - Список тенантов + - Удалить тенанта + - Попытка без admin token → 401 + +2. Изоляция: + - Тенант A создаёт очередь "q1" + - Тенант B создаёт очередь "q1" (ДОЛЖНА создаться — разные тенанты!) + - Тенант A видит только свою q1 + - Тенант B видит только свою q1 + - Тенант A отправляет сообщение → Тенант B НЕ получает его + +3. CRUD: стандартный Create → Send → Receive → Delete flow + +4. Лимиты: тенант с max_queues=2 не может создать 3-ю очередь + +--- + +## 4. ИТОГОВАЯ СТРУКТУРА ПРОЕКТА + +``` +shared-sqs/ +├── app/ +│ ├── cmd/ +│ │ └── main.go # Entry point +│ ├── admin/ +│ │ └── admin_handlers.go # Admin API handlers +│ ├── auth/ +│ │ └── auth_middleware.go # Auth middleware +│ ├── tenant/ +│ │ └── tenant_store.go # Tenant model + in-memory store +│ ├── gosqs/ +│ │ ├── create_queue.go # Modified: tenant-scoped +│ │ ├── send_message.go # Modified: tenant-scoped +│ │ ├── receive_message.go # Modified: tenant-scoped +│ │ ├── delete_message.go # Modified: tenant-scoped +│ │ ├── ... (все handlers) +│ │ ├── tenant_helpers.go # NEW: tenant key/url/arn helpers +│ │ └── gosqs.go # PeriodicTasks (без изменений) +│ ├── models/ +│ │ ├── globals.go # Без SyncTopics +│ │ ├── models.go # Без SNS structs +│ │ ├── configuration.go # Упрощённый +│ │ ├── requests.go # Только SQS requests +│ │ └── responses.go # Только SQS responses +│ ├── router/ +│ │ └── router.go # С middleware и admin routes +│ ├── interfaces/ +│ └── utils/ +├── deployments/ +│ └── k8s/ +│ ├── deployment.yaml +│ ├── service.yaml +│ └── ingress.yaml +├── tests/ +│ └── shared_sqs_test.sh +├── doc/ +│ └── PLAN.md # Этот файл +├── Dockerfile +├── Makefile +├── go.mod +├── go.sum +├── .gitignore +└── README.md +``` + +--- + +## 5. ВСЕ ЛОВУШКИ (СВОДКА) + +| # | Ловушка | Где | Последствие если пропустить | +|---|---------|-----|---------------------------| +| 1 | `crypto/rand` vs `math/rand` для ключей | tenant.go | Предсказуемые ключи = уязвимость | +| 2 | Delete tenant: удалить из ОБОИХ индексов (byID + byAccessKey) | tenant.go | Orphaned данные | +| 3 | AWS CLI отправляет Signature V4 — нужно парсить | middleware.go | SDK не работает | +| 4 | Long polling до 20 сек — не ставить короткий таймаут | middleware.go | ReceiveMessage обрывается | +| 5 | X-Amz-Security-Token — игнорировать | middleware.go | Ложная ошибка auth | +| 6 | URL parsing: последний сегмент = queueName | send_message.go | Берёт tenantID вместо queueName | +| 7 | getQueueFromPath() парсит URL — проверить с новым форматом | gosqs.go | Неправильное имя очереди | +| 8 | FIFO: HasFIFOQueueName() должен получить queueName, не key | create_queue.go | FIFO не работает | +| 9 | DLQ ARN → queue name → tenant prefix | set_queue_attributes.go | Чужая DLQ или not found | +| 10 | QueueUrl в ответах ОБЯЗАН содержать tenantID | create_queue.go, get_queue_url.go | SDK не может send/receive | +| 11 | DELETE tenant → удалить ВСЕ очереди | admin.go | Memory leak | +| 12 | Admin API под отдельной auth (bearer token) | router.go | Тенант = admin | +| 13 | Graceful shutdown (SIGTERM) | main.go | Потеря данных при restart | +| 14 | Deployment strategy: Recreate, НЕ RollingUpdate | deployment.yaml | Split brain | +| 15 | .gitignore: бинарник shared-sqs | .gitignore | Бинарник в git | +| 16 | go mod tidy после чистки SNS | go.mod | Лишние зависимости | +| 17 | Все import paths: goaws → shared-sqs | *.go | Не компилируется | + +--- + +## 6. ПОРЯДОК ВЫПОЛНЕНИЯ + +1. Этап 1 → go build → /health работает +2. Этап 2 → tenant store готов (можно юнит-тестом проверить) +3. Этап 3 → middleware готов +4. Этап 4 → САМЫЙ БОЛЬШОЙ. Делать handler за handler, каждый раз проверяя go build +5. Этап 5 → Admin API +6. Этап 6 → Wiring всего вместе в main.go +7. Этап 7 → Docker + K8s +8. Этап 8 → Makefile +9. Этап 9 → Тесты + +**После каждого этапа: `go build` должен проходить. НЕ НАКАПЛИВАТЬ ошибки компиляции.** + +--- + +## 7. ЧЕГО НЕ ДЕЛАЕМ (scope out) + +- ❌ Persistence (восстановление после рестарта) — in-memory OK для MVP +- ❌ AWS Signature V4 verification — только извлечение AccessKeyId +- ❌ SNS — удаляем полностью +- ❌ HTTPS — TLS на ingress, не в приложении +- ❌ Rate limiting — можно добавить позже +- ❌ Metrics/Prometheus — можно добавить позже +- ❌ UI — нет UI, только API +- ❌ Scale-to-zero — один pod всегда работает +ENDOFPLAN cat > ~/terra/sless/shared-sqs/PLAN.md << 'ENDOFPLAN' +# shared-sqs — План реализации + +**Дата:** 2026-04-09 +**Исполнитель:** Claude Sonnet (или другой агент) +**Подготовил:** Claude Opus 4 (анализ GoAWS, архитектура, ловушки) + +--- + +## 1. ЧТО ЭТО + +Multi-tenant SQS-совместимый сервис на базе форка [GoAWS](https://github.com/Admiral-Piett/goaws) (Go, MIT, 835 stars). + +**Отличие от sqs-operator:** sqs-operator деплоит каждому тенанту ОТДЕЛЬНЫЙ pod с ElasticMQ (~300MB RAM каждый). shared-sqs — ОДИН pod обслуживает ВСЕХ тенантов (~50MB RAM base). + +**Что shared-sqs делает:** +- SQS-совместимый API (CreateQueue, SendMessage, ReceiveMessage, DeleteMessage и т.д.) +- Аутентификация по AccessKeyId (из AWS Authorization header) +- Изоляция очередей между тенантами (тенант видит ТОЛЬКО свои очереди) +- Admin API для управления тенантами (CRUD) +- Работает с AWS CLI и AWS SDK без модификаций + +--- + +## 2. АРХИТЕКТУРА GoAWS (то, что форкаем) + +### 2.1 Структура исходников +``` +app/ +├── cmd/goaws.go # Entry point (~40 LOC): флаги, загрузка конфига, HTTP сервер +├── conf/ # Загрузка YAML конфига +├── gosqs/ # SQS handlers (ЯДРО — ~20 файлов) +│ ├── create_queue.go # CreateQueueV1() +│ ├── send_message.go # SendMessageV1() +│ ├── receive_message.go # ReceiveMessageV1() +│ ├── delete_message.go # DeleteMessageV1() +│ ├── delete_message_batch.go +│ ├── delete_queue.go +│ ├── get_queue_attributes.go +│ ├── get_queue_url.go +│ ├── list_queues.go +│ ├── purge_queue.go +│ ├── send_message_batch.go +│ ├── set_queue_attributes.go +│ ├── change_message_visibility.go +│ ├── queue_attributes.go # Helpers для атрибутов +│ └── gosqs.go # PeriodicTasks (visibility timeout, DLQ, dedup) +├── gosns/ # SNS handlers — НЕ НУЖНЫ, УДАЛИТЬ +├── models/ +│ ├── globals.go # SyncQueues, SyncTopics — глобальные map + RWMutex +│ ├── models.go # Queue, SqsMessage, Topic structs +│ ├── configuration.go # Environment, EnvQueue, config structs +│ ├── constants.go +│ ├── conversions.go # Парсинг тел запросов +│ ├── errors.go # AWS-совместимые ошибки +│ ├── helpers.go +│ ├── requests.go # Request structs (CreateQueueRequest, SendMessageRequest и т.д.) +│ └── responses.go # Response structs (XML + JSON) +├── router/ +│ └── router.go # gorilla/mux, actionHandler, routingTableV1 +├── interfaces/ # AbstractResponseBody interface +├── utils/ # Hash, MD5, REQUEST_TRANSFORMER +├── mocks/ # Тестовые моки +├── fixtures/ # Тестовые данные +├── servertest/ +└── test/ +``` + +### 2.2 Критические архитектурные точки + +**Глобальный state** (`models/globals.go`): +```go +var SyncQueues = struct { + sync.RWMutex + Queues map[string]*Queue +}{Queues: make(map[string]*Queue)} +``` +Все очереди храняться В ОДНОМ map. Ключ = имя очереди (string). + +**Роутинг** (`router/router.go`): +```go +r.HandleFunc("/", actionHandler) +r.HandleFunc("/{account}", actionHandler) +r.HandleFunc("/queue/{queueName}", actionHandler) +r.HandleFunc("/{account}/{queueName}", actionHandler) +``` +Все запросы идут в `actionHandler`, который извлекает `Action` из: +- Query param `Action=CreateQueue` (AWS Query Protocol) +- Header `X-Amz-Target: AmazonSQS.CreateQueue` (AWS JSON Protocol) + +**Dispatch table** (`router/router.go`): +```go +var routingTableV1 = map[string]func(r *http.Request) (int, interfaces.AbstractResponseBody){ + "CreateQueue": sqs.CreateQueueV1, + "SendMessage": sqs.SendMessageV1, + "ReceiveMessage": sqs.ReceiveMessageV1, + // ... все SQS + SNS actions +} +``` + +**URL конструкция** (в create_queue.go): +```go +queueUrl := "http://" + host + ":" + port + "/" + accountID + "/" + queueName +queueArn := "arn:aws:sqs:" + region + ":" + accountID + ":" + queueName +``` +`accountID` берётся из `models.CurrentEnvironment.AccountID` — ГЛОБАЛЬНАЯ переменная (одна на всех). + +**Зависимости** (go.mod): +- `gorilla/mux v1.8.0` — роутер +- `gorilla/schema v1.4.1` — form decoder +- `google/uuid v1.6.0` — UUID генерация +- `sirupsen/logrus` — логирование +- `ghodss/yaml` — YAML парсинг +- `aws/aws-sdk-go v1.47.3` — только для тестов + +--- + +## 3. ПЛАН ИЗМЕНЕНИЙ + +### 3.0 Общие правила работы + +**КРИТИЧНО — все команды ТОЛЬКО через SSH:** +``` +ssh -i /home/naeel/.ssh/naeel_vm_id_ed25519 -o StrictHostKeyChecking=no naeel@5.172.178.213 КОМАНДА +``` + +**Go файлы редактировать ТОЛЬКО через Python patch скрипты на VM**, НЕ через sshfs/VS Code. + +**Markdown/yaml/conf файлы** можно создавать через `cat > file << EOF` по SSH. + +**Перед go build** — убедиться что бинарник в `.gitignore`. + +### Этап 1: Клон GoAWS + чистка (1 час) + +**Действия:** + +1. Клонировать GoAWS в `shared-sqs/`: +```bash +cd ~/terra/sless/shared-sqs +git clone https://github.com/Admiral-Piett/goaws.git _upstream +# Копируем ТОЛЬКО нужное: +cp -r _upstream/app ./app +cp _upstream/go.mod ./go.mod +cp _upstream/go.sum ./go.sum +cp _upstream/Dockerfile ./Dockerfile +rm -rf _upstream +``` + +2. Поменять module name в go.mod: +``` +module shared-sqs +go 1.22 +``` +(Повысить версию Go с 1.18 до 1.22+) + +3. Обновить все import paths: + - Заменить `github.com/Admiral-Piett/goaws/app/` → `shared-sqs/app/` + - Это во ВСЕХ .go файлах + +4. УДАЛИТЬ всё связанное с SNS: + - `app/gosns/` — целиком + - Из `router/router.go` — убрать все SNS записи из `routingTableV1` + - Из `models/globals.go` — убрать `SyncTopics` + - Из `models/models.go` — убрать `Topic`, `Subscription`, `SNSMessage`, `FilterPolicy` + - Из `models/configuration.go` — убрать `EnvTopic`, `EnvSubsciption` + - Из `models/requests.go` и `responses.go` — убрать SNS-related structs + +5. УДАЛИТЬ тестовые/mock директории (мы напишем свои тесты): + - `app/mocks/` + - `app/fixtures/` + - `app/servertest/` + - `app/test/` + - `app/smoke_tests/` (если скопировалась) + +6. Проверить что компилируется: +```bash +cd ~/terra/sless/shared-sqs +go mod tidy +go build -o shared-sqs app/cmd/goaws.go +``` + +7. Проверить что стартует: +```bash +./shared-sqs -debug +# В другом окне: curl http://localhost:4100/health +# Ожидание: 200 OK +``` + +**Тест прохождения этапа:** `go build` успешен, `/health` возвращает 200. + +--- + +### Этап 2: Tenant Model + хранилище (30 мин) + +**Создать файл `app/tenant/tenant.go`:** + +```go +package tenant + +import ( + "crypto/rand" + "encoding/hex" + "sync" + "time" +) + +// Tenant — модель тенанта shared-sqs +type Tenant struct { + ID string // уникальный идентификатор (генерируется) + Name string // имя тенанта (для отображения) + AccessKey string // аналог AWS AccessKeyId + SecretKey string // аналог AWS SecretAccessKey + MaxQueues int // лимит очередей (0 = безлимит) + CreatedAt time.Time + Active bool +} + +// TenantStore — in-memory хранилище тенантов +type TenantStore struct { + sync.RWMutex + // byID — поиск по ID + byID map[string]*Tenant + // byAccessKey — поиск по AccessKey (для auth) + byAccessKey map[string]*Tenant +} + +// NewTenantStore — создаёт пустое хранилище +func NewTenantStore() *TenantStore { ... } + +// Create — создаёт нового тенанта, генерирует ключи +func (s *TenantStore) Create(name string, maxQueues int) (*Tenant, error) { ... } + +// GetByAccessKey — поиск тенанта по AccessKeyId (для auth middleware) +func (s *TenantStore) GetByAccessKey(accessKey string) (*Tenant, bool) { ... } + +// GetByID — поиск по ID +func (s *TenantStore) GetByID(id string) (*Tenant, bool) { ... } + +// Delete — удаление тенанта +func (s *TenantStore) Delete(id string) bool { ... } + +// List — список всех тенантов +func (s *TenantStore) List() []*Tenant { ... } +``` + +**Генерация ключей (БЕЗОПАСНАЯ):** +```go +func generateAccessKey() string { + // Формат: SSAK-{random hex 12} (SS = Shared SQS) + b := make([]byte, 12) + rand.Read(b) + return "SSAK-" + hex.EncodeToString(b) +} + +func generateSecretKey() string { + // 32 байта random → 64 hex символа + b := make([]byte, 32) + rand.Read(b) + return hex.EncodeToString(b) +} +``` + +**ЛОВУШКА #1:** Используй `crypto/rand`, НЕ `math/rand` для ключей. `math/rand` предсказуем. + +**ЛОВУШКА #2:** Два индекса (byID и byAccessKey) — при Delete надо удалить из ОБОИХ. + +**Тест прохождения:** TenantStore создаёт/ищет/удаляет тенантов. Ключи уникальны. + +--- + +### Этап 3: Auth Middleware (30 мин) + +**Создать файл `app/auth/middleware.go`:** + +```go +package auth + +import ( + "context" + "net/http" + "strings" +) + +type contextKey string +const TenantContextKey contextKey = "tenant" + +// AuthMiddleware — извлекает AccessKeyId из AWS Authorization header +// и находит тенанта в store +func AuthMiddleware(store *tenant.TenantStore) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // /health и /admin/** — пропускаем (разная auth) + if r.URL.Path == "/health" { + next.ServeHTTP(w, r) + return + } + if strings.HasPrefix(r.URL.Path, "/admin/") { + next.ServeHTTP(w, r) + return + } + + accessKeyId := extractAccessKeyId(r) + if accessKeyId == "" { + writeAuthError(w, "MissingAuthenticationToken") + return + } + + t, ok := store.GetByAccessKey(accessKeyId) + if !ok || !t.Active { + writeAuthError(w, "InvalidClientTokenId") + return + } + + ctx := context.WithValue(r.Context(), TenantContextKey, t) + next.ServeHTTP(w, r.WithContext(ctx)) + }) + } +} +``` + +**Извлечение AccessKeyId из Authorization header:** +```go +// extractAccessKeyId — извлекает AWS AccessKeyId из запроса +// Формат header: "AWS4-HMAC-SHA256 Credential={AccessKeyId}/{date}/{region}/sqs/aws4_request, ..." +func extractAccessKeyId(r *http.Request) string { + // Вариант 1: Authorization header (AWS Signature V4) + auth := r.Header.Get("Authorization") + if strings.HasPrefix(auth, "AWS4-HMAC-SHA256") { + // Ищем "Credential=" и берём всё до первого "/" + idx := strings.Index(auth, "Credential=") + if idx >= 0 { + rest := auth[idx+len("Credential="):] + slashIdx := strings.Index(rest, "/") + if slashIdx > 0 { + return rest[:slashIdx] + } + } + } + + // Вариант 2: Query parameter (presigned URLs) + if cred := r.URL.Query().Get("X-Amz-Credential"); cred != "" { + parts := strings.SplitN(cred, "/", 2) + if len(parts) > 0 { + return parts[0] + } + } + + return "" +} +``` + +**ЛОВУШКА #3:** AWS CLI ВСЕГДА отправляет Authorization header с Signature V4. Наш middleware должен УМЕТЬ парсить его, даже если мы НЕ ПРОВЕРЯЕМ подпись. + +**ЛОВУШКА #4:** Когда AWS SDK делает `ReceiveMessage` с long polling (WaitTimeSeconds > 0), соединение держится до 20 секунд. Auth middleware не должен ставить таймаут короче. + +**ЛОВУШКА #5:** AWS SDK может отправлять запросы с `X-Amz-Security-Token` (STS). Игнорировать этот header. + +**Тест прохождения:** Запрос с `Authorization: AWS4-HMAC-SHA256 Credential=SSAK-abc123.../...` → тенант найден в контексте. Запрос без header → 403. + +--- + +### Этап 4: Queue Isolation — САМЫЙ КРИТИЧНЫЙ (2 часа) + +Это ЯДРО всех изменений. Все SQS handlers должны работать с tenant-scoped очередями. + +**4.1 Изменение ключей в SyncQueues** + +Сейчас: `SyncQueues.Queues["my-queue"]` +Станет: `SyncQueues.Queues["SSAK-abc123:my-queue"]` + +Формат внутреннего ключа: `{tenantAccessKey}:{queueName}` + +Почему AccessKey а не TenantID: AccessKey уже есть в auth context, не надо лишний lookup. AccessKey уникален. + +**4.2 Helper функции (создать `app/gosqs/tenant_helpers.go`):** + +```go +// tenantQueueKey — внутренний ключ очереди в SyncQueues +func tenantQueueKey(tenantAccessKey, queueName string) string { + return tenantAccessKey + ":" + queueName +} + +// getTenantFromContext — извлекает тенанта из request context +func getTenantFromContext(r *http.Request) *tenant.Tenant { + t, _ := r.Context().Value(auth.TenantContextKey).(*tenant.Tenant) + return t +} + +// tenantQueueUrl — формирует URL очереди для тенанта +func tenantQueueUrl(t *tenant.Tenant, queueName string) string { + return "http://" + models.CurrentEnvironment.Host + ":" + + models.CurrentEnvironment.Port + "/" + t.ID + "/" + queueName +} + +// tenantQueueArn — формирует ARN очереди +func tenantQueueArn(t *tenant.Tenant, queueName string) string { + return "arn:aws:sqs:" + models.CurrentEnvironment.Region + ":" + t.ID + ":" + queueName +} +``` + +**4.3 Изменения в КАЖДОМ handler (подробно)** + +**create_queue.go — CreateQueueV1:** +``` +БЫЛО: + queueName := requestBody.QueueName + key := queueName + url := http://host:port/accountID/queueName + +СТАЛО: + t := getTenantFromContext(req) + queueName := requestBody.QueueName + key := tenantQueueKey(t.AccessKey, queueName) + url := tenantQueueUrl(t, queueName) + arn := tenantQueueArn(t, queueName) + // Проверка лимита очередей: + if t.MaxQueues > 0 { + count := countTenantQueues(t.AccessKey) + if count >= t.MaxQueues { + return error "LimitExceeded" + } + } + models.SyncQueues.Queues[key] = queue +``` + +**send_message.go — SendMessageV1:** +``` +БЫЛО: + queueName = lastSegmentOfUrl(queueUrl) + _, ok := models.SyncQueues.Queues[queueName] + +СТАЛО: + t := getTenantFromContext(req) + queueName = lastSegmentOfUrl(queueUrl) + key := tenantQueueKey(t.AccessKey, queueName) + _, ok := models.SyncQueues.Queues[key] +``` + +**receive_message.go — ReceiveMessageV1:** — аналогично send_message + +**delete_message.go — DeleteMessageV1:** — аналогично + +**delete_message_batch.go — DeleteMessageBatchV1:** — аналогично + +**delete_queue.go — DeleteQueueV1:** +``` +СТАЛО: + t := getTenantFromContext(req) + key := tenantQueueKey(t.AccessKey, queueName) + delete(models.SyncQueues.Queues, key) +``` + +**list_queues.go — ListQueuesV1:** +``` +БЫЛО: + for _, queue := range models.SyncQueues.Queues { + urls = append(urls, queue.URL) + } + +СТАЛО: + t := getTenantFromContext(req) + prefix := t.AccessKey + ":" + for key, queue := range models.SyncQueues.Queues { + if strings.HasPrefix(key, prefix) { + urls = append(urls, queue.URL) + } + } +``` + +**get_queue_url.go — GetQueueUrlV1:** — добавить tenant prefix к lookup + +**get_queue_attributes.go — GetQueueAttributesV1:** — добавить tenant prefix + +**set_queue_attributes.go — SetQueueAttributesV1:** — добавить tenant prefix + +**purge_queue.go — PurgeQueueV1:** — добавить tenant prefix + +**change_message_visibility.go — ChangeMessageVisibilityV1:** — добавить tenant prefix + +**send_message_batch.go — SendMessageBatchV1:** — добавить tenant prefix + +**gosqs.go — PeriodicTasks:** — НЕ ТРОГАТЬ. Он итерирует ВСЕ очереди по полному ключу, не по имени. Работает корректно. + +**ЛОВУШКА #6:** `SendMessageV1` извлекает queueName из QueueUrl через `strings.Split(url, "/")` и берёт ПОСЛЕДНИЙ сегмент. Если URL = `http://host:port/tenantID/myqueue`, последний сегмент = `myqueue` — это ПРАВИЛЬНО, не ломается. + +**ЛОВУШКА #7:** `getQueueFromPath()` в gosqs.go тоже парсит URL. Убедиться что при `/{tenantID}/{queueName}` парсинг берёт queueName, а не tenantID. + +**ЛОВУШКА #8:** FIFO очереди имеют имена вида `myqueue.fifo`. Ключ будет `SSAK-xxx:myqueue.fifo` — это OK, `.fifo` стоит в конце имени, не ключа. Проверить что `utils.HasFIFOQueueName()` получает `queueName`, а не `key`. + +**ЛОВУШКА #9:** RedrivePolicy содержит ARN target очереди. При парсинге ARN в DLQ setup — извлечь имя очереди из ARN, затем добавить tenant prefix для lookup. ОБА (основная и DLQ) должны принадлежать одному тенанту. + +**ЛОВУШКА #10:** `QueueUrl` в ответах CreateQueue и GetQueueUrl используется AWS SDK для всех последующих вызовов. Если формат URL неправильный — SDK не сможет Send/Receive. URL ОБЯЗАН содержать tenantID в пути: `http://host:port/{tenantID}/{queueName}`. + +**4.4 Изменения в роутере** + +В `router.go` — маршрут `/{account}/{queueName}` уже существует. `{account}` = наш `{tenantID}`. Но нужно добавить middleware: + +```go +func New(tenantStore *tenant.TenantStore) http.Handler { + r := mux.NewRouter() + r.HandleFunc("/health", health).Methods("GET") + + // Admin API — отдельная auth (bearer token) + admin := r.PathPrefix("/admin").Subrouter() + // ... admin routes (см. Этап 5) + + // SQS API — tenant auth + sqsRouter := r.PathPrefix("/").Subrouter() + sqsRouter.Use(auth.AuthMiddleware(tenantStore)) + sqsRouter.HandleFunc("/", actionHandler).Methods("GET", "POST") + sqsRouter.HandleFunc("/{account}", actionHandler).Methods("GET", "POST") + sqsRouter.HandleFunc("/queue/{queueName}", actionHandler).Methods("GET", "POST") + sqsRouter.HandleFunc("/{account}/{queueName}", actionHandler).Methods("GET", "POST") + + return r +} +``` + +**Тест прохождения этапа 4:** +```bash +# Создать тенанта через Admin API +curl -X POST http://localhost:4100/admin/tenants -H "Authorization: Bearer $ADMIN_TOKEN" \ + -d max_queues:10 +# → возвращает access_key, secret_key + +# Создать очередь как тенант +aws --endpoint-url http://localhost:4100 sqs create-queue --queue-name test-q +# Настроить AWS credentials = access_key/secret_key из шага выше +# → возвращает QueueUrl с tenantID + +# Отправить и получить сообщение +aws --endpoint-url http://localhost:4100 sqs send-message --queue-url $QUEUE_URL --message-body "hello" +aws --endpoint-url http://localhost:4100 sqs receive-message --queue-url $QUEUE_URL +# → сообщение получено + +# Второй тенант НЕ видит очереди первого +# Со вторым access_key: +aws --endpoint-url http://localhost:4100 sqs list-queues +# → пустой список +``` + +--- + +### Этап 5: Admin API (30 мин) + +**Создать файл `app/admin/admin.go`:** + +Эндпоинты: +``` +POST /admin/tenants — создать тенанта +GET /admin/tenants — список тенантов +GET /admin/tenants/{id} — детали тенанта +DELETE /admin/tenants/{id} — удалить тенанта (и ВСЕ его очереди!) +GET /admin/health — детальный health (кол-во тенантов, очередей, сообщений) +``` + +**Auth для Admin API:** Header `Authorization: Bearer {admin_token}`. Admin token задаётся через: +- Переменная окружения `SHARED_SQS_ADMIN_TOKEN` +- Или в конфиге YAML + +**ЛОВУШКА #11:** При DELETE тенанта — ОБЯЗАТЕЛЬНО удалить ВСЕ его очереди из SyncQueues. Иначе memory leak. Итерировать SyncQueues.Queues, удалить все ключи с prefix `{accessKey}:`. + +**ЛОВУШКА #12:** Admin API НЕ должен быть доступен через тот же ingress что SQS API. Либо другой path prefix, либо другой порт. Рекомендация: path prefix `/admin/`, защищённый bearer token. НЕ ЗАБЫТЬ в auth middleware пропускать `/admin/` пути. + +**Request/Response форматы:** + +POST /admin/tenants: +```json +Request: {"name": "customer-001", "max_queues": 50} +Response: {"id": "t-a1b2c3", "name": "customer-001", "access_key": "SSAK-...", "secret_key": "...", "max_queues": 50} +``` +Важно: secret_key показывается ТОЛЬКО при создании. В List/Get — не включать. + +--- + +### Этап 6: Entry Point + Configuration (20 мин) + +**Модифицировать `app/cmd/goaws.go` (переименовать в `app/cmd/main.go`):** + +```go +func main() { + // Флаги + var configFile string + var adminToken string + var port string + flag.StringVar(&configFile, "config", "", "config file") + flag.StringVar(&adminToken, "admin-token", "", "admin API token") + flag.StringVar(&port, "port", "4100", "listen port") + flag.Parse() + + // Admin token: flag > env > config + if adminToken == "" { + adminToken = os.Getenv("SHARED_SQS_ADMIN_TOKEN") + } + if adminToken == "" { + log.Fatal("admin token required: use --admin-token or SHARED_SQS_ADMIN_TOKEN env") + } + + // Инициализация + tenantStore := tenant.NewTenantStore() + + // Загрузка конфига (если указан) — может содержать pre-created тенантов + if configFile != "" { + conf.LoadConfig(configFile, tenantStore) + } + + // Роутер + r := router.New(tenantStore, adminToken) + + // Periodic tasks + quit := make(chan bool) + go gosqs.PeriodicTasks(1*time.Second, quit) + + // Graceful shutdown + // ... (signal handling, quit channel) + + log.Infof("shared-sqs listening on 0.0.0.0:%s", port) + log.Fatal(http.ListenAndServe("0.0.0.0:"+port, r)) +} +``` + +**ЛОВУШКА #13:** GoAWS не обрабатывает сигналы. ДОБАВИТЬ graceful shutdown (SIGTERM/SIGINT → закрыть quit channel → остановить PeriodicTasks → дождаться завершения). + +--- + +### Этап 7: Dockerfile + Kubernetes (30 мин) + +**Dockerfile:** +```dockerfile +FROM golang:1.22-alpine AS builder +WORKDIR /build +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -o shared-sqs app/cmd/main.go + +FROM alpine:3.19 +RUN apk --no-cache add ca-certificates +COPY --from=builder /build/shared-sqs /usr/local/bin/shared-sqs +EXPOSE 4100 +ENTRYPOINT ["shared-sqs"] +``` + +**Kubernetes manifests (`deployments/k8s/`):** + +deployment.yaml: +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: shared-sqs + namespace: shared-sqs +spec: + replicas: 1 + strategy: + type: Recreate # НЕ RollingUpdate! Урок из ERR-SQS-06. + selector: + matchLabels: + app: shared-sqs + template: + spec: + containers: + - name: shared-sqs + image: pearlharbor.registryk8s.services.ngcloud.ru/naeel/shared-sqs:v0.1.0 + ports: + - containerPort: 4100 + env: + - name: SHARED_SQS_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: shared-sqs-admin + key: token + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 4100 + readinessProbe: + httpGet: + path: /health + port: 4100 +``` + +**ЛОВУШКА #14:** strategy: Recreate, НЕ RollingUpdate. In-memory state не шарится между подами. При RollingUpdate новый pod стартует с пустым state, а старый ещё жив = split brain. + +service.yaml, ingress.yaml — стандартные. + +**Реестр:** `pearlharbor.registryk8s.services.ngcloud.ru/naeel/shared-sqs` + +--- + +### Этап 8: Makefile (10 мин) + +```makefile +IMAGE_REPO=pearlharbor.registryk8s.services.ngcloud.ru/naeel/shared-sqs +VERSION=v0.1.0 + +build: +CGO_ENABLED=0 go build -o shared-sqs app/cmd/main.go + +docker-build: +docker build -t $(IMAGE_REPO):$(VERSION) . + +docker-push: +docker push $(IMAGE_REPO):$(VERSION) + +test: +go test ./... + +run: +./shared-sqs --admin-token=dev-token-123 --port=4100 -debug + +clean: +rm -f shared-sqs +``` + +--- + +### Этап 9: Тесты (1 час) + +**Создать `tests/shared_sqs_test.sh`** — bash тесты (как в customer-sqs): + +1. Admin API: + - Создать тенанта + - Список тенантов + - Удалить тенанта + - Попытка без admin token → 401 + +2. Изоляция: + - Тенант A создаёт очередь "q1" + - Тенант B создаёт очередь "q1" (ДОЛЖНА создаться — разные тенанты!) + - Тенант A видит только свою q1 + - Тенант B видит только свою q1 + - Тенант A отправляет сообщение → Тенант B НЕ получает его + +3. CRUD: стандартный Create → Send → Receive → Delete flow + +4. Лимиты: тенант с max_queues=2 не может создать 3-ю очередь + +--- + +## 4. ИТОГОВАЯ СТРУКТУРА ПРОЕКТА + +``` +shared-sqs/ +├── app/ +│ ├── cmd/ +│ │ └── main.go # Entry point +│ ├── admin/ +│ │ └── admin_handlers.go # Admin API handlers +│ ├── auth/ +│ │ └── auth_middleware.go # Auth middleware +│ ├── tenant/ +│ │ └── tenant_store.go # Tenant model + in-memory store +│ ├── gosqs/ +│ │ ├── create_queue.go # Modified: tenant-scoped +│ │ ├── send_message.go # Modified: tenant-scoped +│ │ ├── receive_message.go # Modified: tenant-scoped +│ │ ├── delete_message.go # Modified: tenant-scoped +│ │ ├── ... (все handlers) +│ │ ├── tenant_helpers.go # NEW: tenant key/url/arn helpers +│ │ └── gosqs.go # PeriodicTasks (без изменений) +│ ├── models/ +│ │ ├── globals.go # Без SyncTopics +│ │ ├── models.go # Без SNS structs +│ │ ├── configuration.go # Упрощённый +│ │ ├── requests.go # Только SQS requests +│ │ └── responses.go # Только SQS responses +│ ├── router/ +│ │ └── router.go # С middleware и admin routes +│ ├── interfaces/ +│ └── utils/ +├── deployments/ +│ └── k8s/ +│ ├── deployment.yaml +│ ├── service.yaml +│ └── ingress.yaml +├── tests/ +│ └── shared_sqs_test.sh +├── doc/ +│ └── PLAN.md # Этот файл +├── Dockerfile +├── Makefile +├── go.mod +├── go.sum +├── .gitignore +└── README.md +``` + +--- + +## 5. ВСЕ ЛОВУШКИ (СВОДКА) + +| # | Ловушка | Где | Последствие если пропустить | +|---|---------|-----|---------------------------| +| 1 | `crypto/rand` vs `math/rand` для ключей | tenant.go | Предсказуемые ключи = уязвимость | +| 2 | Delete tenant: удалить из ОБОИХ индексов (byID + byAccessKey) | tenant.go | Orphaned данные | +| 3 | AWS CLI отправляет Signature V4 — нужно парсить | middleware.go | SDK не работает | +| 4 | Long polling до 20 сек — не ставить короткий таймаут | middleware.go | ReceiveMessage обрывается | +| 5 | X-Amz-Security-Token — игнорировать | middleware.go | Ложная ошибка auth | +| 6 | URL parsing: последний сегмент = queueName | send_message.go | Берёт tenantID вместо queueName | +| 7 | getQueueFromPath() парсит URL — проверить с новым форматом | gosqs.go | Неправильное имя очереди | +| 8 | FIFO: HasFIFOQueueName() должен получить queueName, не key | create_queue.go | FIFO не работает | +| 9 | DLQ ARN → queue name → tenant prefix | set_queue_attributes.go | Чужая DLQ или not found | +| 10 | QueueUrl в ответах ОБЯЗАН содержать tenantID | create_queue.go, get_queue_url.go | SDK не может send/receive | +| 11 | DELETE tenant → удалить ВСЕ очереди | admin.go | Memory leak | +| 12 | Admin API под отдельной auth (bearer token) | router.go | Тенант = admin | +| 13 | Graceful shutdown (SIGTERM) | main.go | Потеря данных при restart | +| 14 | Deployment strategy: Recreate, НЕ RollingUpdate | deployment.yaml | Split brain | +| 15 | .gitignore: бинарник shared-sqs | .gitignore | Бинарник в git | +| 16 | go mod tidy после чистки SNS | go.mod | Лишние зависимости | +| 17 | Все import paths: goaws → shared-sqs | *.go | Не компилируется | + +--- + +## 6. ПОРЯДОК ВЫПОЛНЕНИЯ + +1. Этап 1 → go build → /health работает +2. Этап 2 → tenant store готов (можно юнит-тестом проверить) +3. Этап 3 → middleware готов +4. Этап 4 → САМЫЙ БОЛЬШОЙ. Делать handler за handler, каждый раз проверяя go build +5. Этап 5 → Admin API +6. Этап 6 → Wiring всего вместе в main.go +7. Этап 7 → Docker + K8s +8. Этап 8 → Makefile +9. Этап 9 → Тесты + +**После каждого этапа: `go build` должен проходить. НЕ НАКАПЛИВАТЬ ошибки компиляции.** + +--- + +## 7. ЧЕГО НЕ ДЕЛАЕМ (scope out) + +- ❌ Persistence (восстановление после рестарта) — in-memory OK для MVP +- ❌ AWS Signature V4 verification — только извлечение AccessKeyId +- ❌ SNS — удаляем полностью +- ❌ HTTPS — TLS на ingress, не в приложении +- ❌ Rate limiting — можно добавить позже +- ❌ Metrics/Prometheus — можно добавить позже +- ❌ UI — нет UI, только API +- ❌ Scale-to-zero — один pod всегда работает diff --git a/shared-sqs/app/cmd/goaws.go b/shared-sqs/app/cmd/goaws.go new file mode 100644 index 0000000..ed1c67e --- /dev/null +++ b/shared-sqs/app/cmd/goaws.go @@ -0,0 +1,80 @@ +package main + +import ( + "flag" + "net/http" + "os" + "time" + + "shared-sqs/app/models" + + log "github.com/sirupsen/logrus" + + "shared-sqs/app/conf" + "shared-sqs/app/gosqs" + "shared-sqs/app/router" +) + +func main() { + var filename string + var debug bool + var loglevel string + flag.StringVar(&filename, "config", "", "config file location + name") + flag.BoolVar(&debug, "debug", false, "set debug log level") + flag.StringVar(&loglevel, "loglevel", "info", "log level (default info)") + flag.Parse() + + log.SetFormatter(&log.JSONFormatter{}) + log.SetOutput(os.Stdout) + + if debug { + log.SetLevel(log.DebugLevel) + } else { + level, err := log.ParseLevel(loglevel) + if err != nil { + log.SetLevel(log.InfoLevel) + log.Warnf("Failed to parse loglevel %v, defaulting to info", loglevel) + } else { + log.SetLevel(level) + } + } + + env := "Local" + if flag.NArg() > 0 { + env = flag.Arg(0) + } + + portNumbers := conf.LoadYamlConfig(filename, env) + + if models.CurrentEnvironment.LogToFile { + filename := models.CurrentEnvironment.LogFile + file, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err == nil { + log.SetOutput(file) + } else { + log.Infof("Failed to log to file: %s, using default stderr", filename) + } + } + + r := router.New() + + quit := make(chan bool, 0) + go gosqs.PeriodicTasks(1*time.Second, quit) + + if len(portNumbers) == 1 { + log.Warnf("GoAws listening on: 0.0.0.0:%s", portNumbers[0]) + err := http.ListenAndServe("0.0.0.0:"+portNumbers[0], r) + log.Fatal(err) + } else if len(portNumbers) == 2 { + go func() { + log.Warnf("GoAws listening on: 0.0.0.0:%s", portNumbers[0]) + err := http.ListenAndServe("0.0.0.0:"+portNumbers[0], r) + log.Fatal(err) + }() + log.Warnf("GoAws listening on: 0.0.0.0:%s", portNumbers[1]) + err := http.ListenAndServe("0.0.0.0:"+portNumbers[1], r) + log.Fatal(err) + } else { + log.Fatal("Not enough or too many ports defined to start GoAws.") + } +} diff --git a/shared-sqs/app/conf/config.go b/shared-sqs/app/conf/config.go new file mode 100644 index 0000000..764d579 --- /dev/null +++ b/shared-sqs/app/conf/config.go @@ -0,0 +1,194 @@ +package conf + +import ( +"encoding/json" +"fmt" +"io/fs" +"os" +"path/filepath" +"strconv" +"strings" +"time" + +"shared-sqs/app/models" +"shared-sqs/app/utils" + +log "github.com/sirupsen/logrus" + +"github.com/ghodss/yaml" +) + +var envs map[string]models.Environment + +func LoadYamlConfig(filename string, env string) []string { +ports := []string{"4100"} + +if filename == "" { +root, _ := filepath.Abs(".") +err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { +if "goaws.yaml" == d.Name() { +filename = path +} +return nil +}) +if err != nil || filename == "" { +log.Warn("Failure to find default config file") +return ports +} +} + +filename, _ = filepath.Abs(filename) +if _, err := os.Stat(filename); err != nil { +log.Warnf("Failure to find config file: %s", filename) +return ports +} + +log.Infof("Loading config file: %s", filename) +yamlFile, err := os.ReadFile(filename) +if err != nil { +return ports +} + +err = yaml.Unmarshal(yamlFile, &envs) +if err != nil { +log.Errorf("err: %v\n", err) +return ports +} +if env == "" { +env = "Local" +} + +if envs[env].Region == "" { +models.CurrentEnvironment.Region = "local" +} + +models.CurrentEnvironment = envs[env] + +if envs[env].Port != "" { +ports = []string{envs[env].Port} +} + +models.LogMessages = false +models.LogFile = "./goaws_messages.log" +if envs[env].LogToFile == true { +models.LogMessages = true +if envs[env].LogFile != "" { +models.LogFile = envs[env].LogFile +} +} + +if models.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout <= 0 { +models.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout = 30 +} + +if models.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize <= 0 { +models.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize = 262144 // 256K +} + +if models.CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod <= 0 { +models.CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod = 345600 // 4 days +} + +if models.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds <= 0 { +models.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds = 0 +} + +if models.CurrentEnvironment.AccountID == "" { +models.CurrentEnvironment.AccountID = "queue" +} + +if models.CurrentEnvironment.Host == "" { +models.CurrentEnvironment.Host = "localhost" +models.CurrentEnvironment.Port = "4100" +} + +models.SyncQueues.Lock() +for _, queue := range envs[env].Queues { +queueUrl := "http://" + models.CurrentEnvironment.Host + ":" + models.CurrentEnvironment.Port + +"/" + models.CurrentEnvironment.AccountID + "/" + queue.Name +if models.CurrentEnvironment.Region != "" { +queueUrl = "http://" + models.CurrentEnvironment.Region + "." + models.CurrentEnvironment.Host + ":" + +models.CurrentEnvironment.Port + "/" + models.CurrentEnvironment.AccountID + "/" + queue.Name +} +queueArn := "arn:aws:sqs:" + models.CurrentEnvironment.Region + ":" + models.CurrentEnvironment.AccountID + ":" + queue.Name + +if queue.ReceiveMessageWaitTimeSeconds == 0 { +queue.ReceiveMessageWaitTimeSeconds = models.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds +} +if queue.MaximumMessageSize == 0 { +queue.MaximumMessageSize = models.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize +} +if queue.VisibilityTimeout == 0 { +queue.VisibilityTimeout = models.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout +} +if queue.MessageRetentionPeriod == 0 { +queue.MessageRetentionPeriod = models.CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod +} + +models.SyncQueues.Queues[queue.Name] = &models.Queue{ +Name: queue.Name, +VisibilityTimeout: queue.VisibilityTimeout, +Arn: queueArn, +URL: queueUrl, +ReceiveMessageWaitTimeSeconds: queue.ReceiveMessageWaitTimeSeconds, +MaximumMessageSize: queue.MaximumMessageSize, +MessageRetentionPeriod: queue.MessageRetentionPeriod, +IsFIFO: utils.HasFIFOQueueName(queue.Name), +EnableDuplicates: models.CurrentEnvironment.EnableDuplicates, +Duplicates: make(map[string]time.Time), +} +} + +// Второй проход — устанавливаем RedrivePolicy, чтобы DLQ были доступны независимо от порядка +for _, queue := range envs[env].Queues { +q := models.SyncQueues.Queues[queue.Name] +if queue.RedrivePolicy != "" { +err := setQueueRedrivePolicy(models.SyncQueues.Queues, q, queue.RedrivePolicy) +if err != nil { +log.Errorf("err: %s", err) +return ports +} +} +} + +models.SyncQueues.Unlock() + +return ports +} + +func setQueueRedrivePolicy(queues map[string]*models.Queue, q *models.Queue, strRedrivePolicy string) error { +// Поддерживаем maxReceiveCount как int и как string (AWS SDK использует string) +redrivePolicy1 := struct { +MaxReceiveCount int `json:"maxReceiveCount"` +DeadLetterTargetArn string `json:"deadLetterTargetArn"` +}{} +redrivePolicy2 := struct { +MaxReceiveCount string `json:"maxReceiveCount"` +DeadLetterTargetArn string `json:"deadLetterTargetArn"` +}{} +err1 := json.Unmarshal([]byte(strRedrivePolicy), &redrivePolicy1) +err2 := json.Unmarshal([]byte(strRedrivePolicy), &redrivePolicy2) +maxReceiveCount := redrivePolicy1.MaxReceiveCount +deadLetterQueueArn := redrivePolicy1.DeadLetterTargetArn +if err1 != nil && err2 != nil { +return fmt.Errorf("invalid json for queue redrive policy ") +} else if err1 != nil { +maxReceiveCount, _ = strconv.Atoi(redrivePolicy2.MaxReceiveCount) +deadLetterQueueArn = redrivePolicy2.DeadLetterTargetArn +} + +if (deadLetterQueueArn != "" && maxReceiveCount == 0) || +(deadLetterQueueArn == "" && maxReceiveCount != 0) { +return fmt.Errorf("invalid redrive policy values") +} +dlt := strings.Split(deadLetterQueueArn, ":") +deadLetterQueueName := dlt[len(dlt)-1] +deadLetterQueue, ok := queues[deadLetterQueueName] +if !ok { +return fmt.Errorf("deadletter queue not found") +} +q.DeadLetterQueue = deadLetterQueue +q.MaxReceiveCount = maxReceiveCount + +return nil +} diff --git a/shared-sqs/app/conf/config_test.go b/shared-sqs/app/conf/config_test.go new file mode 100644 index 0000000..f7d8053 --- /dev/null +++ b/shared-sqs/app/conf/config_test.go @@ -0,0 +1,166 @@ +package conf + +import ( + "testing" + + "shared-sqs/app/models" + + "github.com/stretchr/testify/assert" +) + +func TestConfig_NoQueuesOrTopics(t *testing.T) { + env := "NoQueuesOrTopics" + port := LoadYamlConfig("./mock-data/mock-config.yaml", env) + if port[0] != "4100" { + t.Errorf("Expected port number 4200 but got %s\n", port) + } + + numQueues := len(envs[env].Queues) + if numQueues != 0 { + t.Errorf("Expected zero queues to be in the environment but got %d\n", numQueues) + } + numQueues = len(models.SyncQueues.Queues) + if numQueues != 0 { + t.Errorf("Expected zero queues to be in the sqs topics but got %d\n", numQueues) + } + + numTopics := len(envs[env].Topics) + if numTopics != 0 { + t.Errorf("Expected zero topics to be in the environment but got %d\n", numTopics) + } + numTopics = len(models.SyncTopics.Topics) + if numTopics != 0 { + t.Errorf("Expected zero topics to be in the sns topics but got %d\n", numTopics) + } +} + +func TestConfig_CreateQueuesTopicsAndSubscriptions(t *testing.T) { + env := "Local" + port := LoadYamlConfig("./mock-data/mock-config.yaml", env) + if port[0] != "4100" { + t.Errorf("Expected port number 4100 but got %s\n", port) + } + + numQueues := len(envs[env].Queues) + if numQueues != 4 { + t.Errorf("Expected three queues to be in the environment but got %d\n", numQueues) + } + numQueues = len(models.SyncQueues.Queues) + if numQueues != 6 { + t.Errorf("Expected five queues to be in the sqs topics but got %d\n", numQueues) + } + + numTopics := len(envs[env].Topics) + if numTopics != 2 { + t.Errorf("Expected two topics to be in the environment but got %d\n", numTopics) + } + numTopics = len(models.SyncTopics.Topics) + if numTopics != 2 { + t.Errorf("Expected two topics to be in the sns topics but got %d\n", numTopics) + } +} + +func TestConfig_QueueAttributes(t *testing.T) { + var emptyQueue *models.Queue + env := "Local" + port := LoadYamlConfig("./mock-data/mock-config.yaml", env) + if port[0] != "4100" { + t.Errorf("Expected port number 4100 but got %s\n", port) + } + + assert.Equal(t, 10, models.SyncQueues.Queues["local-queue1"].ReceiveMessageWaitTimeSeconds) + assert.Equal(t, 10, models.SyncQueues.Queues["local-queue1"].VisibilityTimeout) + assert.Equal(t, 1024, models.SyncQueues.Queues["local-queue1"].MaximumMessageSize) + assert.Equal(t, emptyQueue, models.SyncQueues.Queues["local-queue1"].DeadLetterQueue) + assert.Equal(t, 0, models.SyncQueues.Queues["local-queue1"].MaxReceiveCount) + assert.Equal(t, 345600, models.SyncQueues.Queues["local-queue1"].MessageRetentionPeriod) + assert.Equal(t, 100, models.SyncQueues.Queues["local-queue3"].MaxReceiveCount) + + assert.Equal(t, "local-queue3-dlq", models.SyncQueues.Queues["local-queue3"].DeadLetterQueue.Name) + assert.Equal(t, 128, models.SyncQueues.Queues["local-queue2"].MaximumMessageSize) + assert.Equal(t, 150, models.SyncQueues.Queues["local-queue2"].VisibilityTimeout) + assert.Equal(t, 245600, models.SyncQueues.Queues["local-queue2"].MessageRetentionPeriod) +} + +func TestConfig_NoQueueAttributeDefaults(t *testing.T) { + env := "NoQueueAttributeDefaults" + LoadYamlConfig("./mock-data/mock-config.yaml", env) + + receiveWaitTime := models.SyncQueues.Queues["local-queue1"].ReceiveMessageWaitTimeSeconds + if receiveWaitTime != 0 { + t.Errorf("Expected local-queue1 Queue to be configured with ReceiveMessageWaitTimeSeconds: 0 but got %d\n", receiveWaitTime) + } + timeoutSecs := models.SyncQueues.Queues["local-queue1"].VisibilityTimeout + if timeoutSecs != 30 { + t.Errorf("Expected local-queue1 Queue to be configured with VisibilityTimeout: 30 but got %d\n", timeoutSecs) + } + + receiveWaitTime = models.SyncQueues.Queues["local-queue2"].ReceiveMessageWaitTimeSeconds + if receiveWaitTime != 20 { + t.Errorf("Expected local-queue2 Queue to be configured with ReceiveMessageWaitTimeSeconds: 20 but got %d\n", receiveWaitTime) + } + + messageRetentionPeriod := models.SyncQueues.Queues["local-queue1"].MessageRetentionPeriod + if messageRetentionPeriod != 345600 { + t.Errorf("Expected local-queue2 Queue to be configured with VisibilityTimeout: 150 but got %d\n", timeoutSecs) + } +} + +func TestConfig_invalid_config_resorts_to_default_queue_attributes(t *testing.T) { + env := "missing" + port := LoadYamlConfig("./mock-data/mock-config.yaml", env) + if port[0] != "4100" { + t.Errorf("Expected port number 4100 but got %s\n", port) + } + + assert.Equal(t, 262144, models.CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize) + assert.Equal(t, 345600, models.CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod) + assert.Equal(t, 0, models.CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, 30, models.CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout) +} + +func TestConfig_LoadYamlConfig_finds_default_config(t *testing.T) { + expectedQueues := []string{ + "local-queue1", + "local-queue2", + "local-queue3", + "local-queue3-dlq", + "local-queue4", + } + expectedTopics := []string{ + "local-topic1", + "local-topic2", + "local-topic3", + "local-topic4", + } + + env := "Local" + LoadYamlConfig("", env) + + queues := models.SyncQueues.Queues + topics := models.SyncTopics.Topics + for _, expectedName := range expectedQueues { + _, ok := queues[expectedName] + assert.True(t, ok) + } + for _, expectedName := range expectedTopics { + _, ok := topics[expectedName] + assert.True(t, ok) + } +} + +func TestConfig_LoadYamlConfig_missing_config_loads_nothing(t *testing.T) { + models.CurrentEnvironment = models.Environment{} + ports := LoadYamlConfig("/garbage", "Local") + + assert.Equal(t, []string{"4100"}, ports) + assert.Equal(t, models.CurrentEnvironment, models.Environment{}) +} + +func TestConfig_LoadYamlConfig_invalid_config_loads_nothing(t *testing.T) { + models.CurrentEnvironment = models.Environment{} + ports := LoadYamlConfig("../common/common.go", "Local") + + assert.Equal(t, []string{"4100"}, ports) + assert.Equal(t, models.CurrentEnvironment, models.Environment{}) +} diff --git a/shared-sqs/app/conf/goaws.yaml b/shared-sqs/app/conf/goaws.yaml new file mode 100755 index 0000000..a9eb77b --- /dev/null +++ b/shared-sqs/app/conf/goaws.yaml @@ -0,0 +1,66 @@ +Local: # Environment name that can be passed on the command line + # (i.e.: ./goaws [Local | Dev] -- defaults to 'Local') + Host: goaws.com # hostname of the goaws system (for docker-compose this is the tag name of the container) + # you can now use either 1 port for both sns and sqs or alternatively you can comment out Port and use SqsPort + SnsPort for compatibilyt with + # yopa and (fage-sns + face-sqs). If both ways are in the config file on the one "Port" will be used by GoAws + Port: 4100 # port to listen on. + # SqsPort: 9324 # alterante Sqs Port + # SnsPort: 9292 # alternate Sns Port + Region: us-east-1 + AccountId: "100010001000" + LogToFile: false # Log messages (true/false) + LogFile: .st/goaws_messages.log # Log filename (for message logging + EnableDuplicates: false # Enable or not deduplication based on messageDeduplicationId + QueueAttributeDefaults: # default attributes for all queues + VisibilityTimeout: 30 # message visibility timeout + ReceiveMessageWaitTimeSeconds: 0 # receive message max wait time + MaximumMessageSize: 262144 # maximum message size (bytes) +# MessageRetentionPeriod: 445600 # time period to retain messages (seconds) NOTE: Functionality not implemented + Queues: # List of queues to create at startup + - Name: local-queue1 # Queue name + - Name: local-queue2 # Queue name + ReceiveMessageWaitTimeSeconds: 20 # Queue receive message max wait time + - Name: local-queue3 # Queue name + RedrivePolicy: '{"maxReceiveCount": 100, "deadLetterTargetArn":"arn:aws:sqs:us-east-1:100010001000:local-queue3-dlq"}' + - Name: local-queue3-dlq # Queue name + Topics: # List of topic to create at startup + - Name: local-topic1 # Topic name - with some Subscriptions + Subscriptions: # List of Subscriptions to create for this topic (queues will be created as required) + - QueueName: local-queue3 # Queue name + Raw: false # Raw message delivery (true/false) + - QueueName: local-queue4 # Queue name + Raw: true # Raw message delivery (true/false) + #FilterPolicy: '{"foo": ["bar"]}' # Subscription's FilterPolicy, json object as a string + - Name: local-topic2 # Topic name - no Subscriptions + - Name: local-topic3 # Topic name - http subscription + Subscriptions: + - Protocol: https + EndPoint: https://enkrogwitfcgi.x.pipedream.net + TopicArn: arn:aws:sns:us-east-1:100010001000:local-topic2 + FilterPolicy: '{"event": ["my_event"]}' + Raw: true + - Name: local-topic4 + RandomLatency: # Parameters for introducing random latency into message queuing + Min: 0 # Desired latency in milliseconds, if min and max are zero, no latency will be applied. + Max: 0 # Desired latency in milliseconds + +Dev: # Another environment + Host: localhost + Port: 4100 + # SqsPort: 9324 + # SnsPort: 9292 + AccountId: "794373491471" + LogToFile: false + LogFile: ./goaws_messages.log + Queues: + - Name: dev-queue1 + - Name: dev-queue2 + Topics: + - Name: dev-topic1 + Subscriptions: + - QueueName: dev-queue3 + Raw: false + - QueueName: dev-queue4 + Raw: true + - Name: dev-topic2 + diff --git a/shared-sqs/app/conf/mock-data/mock-config.yaml b/shared-sqs/app/conf/mock-data/mock-config.yaml new file mode 100644 index 0000000..d3215cf --- /dev/null +++ b/shared-sqs/app/conf/mock-data/mock-config.yaml @@ -0,0 +1,79 @@ +Local: + Host: localhost + Port: 4100 + Region: us-east-1 + AccountId: "100010001000" + LogMessages: true + LogFile: ./goaws_messages.log + QueueAttributeDefaults: + VisibilityTimeout: 10 + ReceiveMessageWaitTimeSeconds: 10 + MaximumMessageSize: 1024 + Queues: + - Name: local-queue1 + - Name: local-queue2 + ReceiveMessageWaitTimeSeconds: 20 + MaximumMessageSize: 128 + VisibilityTimeout: 150 + MessageRetentionPeriod: 245600 + - Name: local-queue3 + RedrivePolicy: '{"maxReceiveCount": 100, "deadLetterTargetArn":"arn:aws:sqs:us-east-1:100010001000:local-queue3-dlq"}' + - Name: local-queue3-dlq + Topics: + - Name: local-topic1 + Subscriptions: + - QueueName: local-queue4 + Raw: false + - QueueName: local-queue5 + Raw: true + FilterPolicy: '{"foo":["bar"]}' + - Name: local-topic2 + +NoQueuesOrTopics: + Host: localhost + Port: 4100 + LogMessages: true + LogFile: ./goaws_messages.log + Region: eu-west-1 + +NoQueueAttributeDefaults: + Host: localhost + Port: 4100 + LogMessages: true + LogFile: ./goaws_messages.log + Region: eu-west-1 + Queues: + - Name: local-queue1 + - Name: local-queue2 + ReceiveMessageWaitTimeSeconds: 20 + +BaseUnitTests: + Host: host + Port: port + Region: region + AccountId: accountID + LogMessages: true + LogFile: ./goaws_messages.log + Queues: + - Name: unit-queue1 + - Name: unit-queue2 + RedrivePolicy: '{"maxReceiveCount": 1, "deadLetterTargetArn":"arn:aws:sqs:us-east-1:100010001000:dead-letter-queue1"}' + - Name: dead-letter-queue1 + - Name: subscribed-queue1 + - Name: subscribed-queue3 + Topics: + - Name: unit-topic1 + Subscriptions: + - QueueName: subscribed-queue1 + Raw: true + - Name: unit-topic2 + - Name: unit-topic3 + Subscriptions: + - QueueName: subscribed-queue3 + Raw: false + - Name: unit-topic-http + Subscriptions: + - Protocol: http + EndPoint: http://over.ride.me/for/tests + TopicArn: arn:aws:sqs:region:accountID:unit-topic-http + Raw: true diff --git a/shared-sqs/app/gosqs/change_message_visibility.go b/shared-sqs/app/gosqs/change_message_visibility.go new file mode 100644 index 0000000..56427c2 --- /dev/null +++ b/shared-sqs/app/gosqs/change_message_visibility.go @@ -0,0 +1,81 @@ +package gosqs + +import ( + "net/http" + "strings" + "time" + + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" +) + +func ChangeMessageVisibilityV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewChangeMessageVisibilityRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - ChangeMessageVisibilityV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + vars := mux.Vars(req) + + queueUrl := requestBody.QueueUrl + queueName := "" + if queueUrl == "" { + queueName = vars["queueName"] + } else { + uriSegments := strings.Split(queueUrl, "/") + queueName = uriSegments[len(uriSegments)-1] + } + + receiptHandle := requestBody.ReceiptHandle + + visibilityTimeout := requestBody.VisibilityTimeout + if visibilityTimeout > 43200 { + return utils.CreateErrorResponseV1("ValidationError", true) + } + + if _, ok := models.SyncQueues.Queues[queueName]; !ok { + return utils.CreateErrorResponseV1("QueueNotFound", true) + } + + models.SyncQueues.Lock() + messageFound := false + for i := 0; i < len(models.SyncQueues.Queues[queueName].Messages); i++ { + queue := models.SyncQueues.Queues[queueName] + msgs := queue.Messages + if msgs[i].ReceiptHandle == receiptHandle { + timeout := models.SyncQueues.Queues[queueName].VisibilityTimeout + if visibilityTimeout == 0 { + msgs[i].ReceiptTime = time.Now().UTC() + msgs[i].ReceiptHandle = "" + msgs[i].VisibilityTimeout = time.Now().Add(time.Duration(timeout) * time.Second) + msgs[i].Retry++ + if queue.MaxReceiveCount > 0 && + queue.DeadLetterQueue != nil && + msgs[i].Retry >= queue.MaxReceiveCount { + queue.DeadLetterQueue.Messages = append(queue.DeadLetterQueue.Messages, msgs[i]) + queue.Messages = append(queue.Messages[:i], queue.Messages[i+1:]...) + } + } else { + msgs[i].VisibilityTimeout = time.Now().Add(time.Duration(visibilityTimeout) * time.Second) + } + messageFound = true + break + } + } + models.SyncQueues.Unlock() + if !messageFound { + return utils.CreateErrorResponseV1("MessageNotInFlight", true) + } + + respStruct := models.ChangeMessageVisibilityResult{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + + return http.StatusOK, &respStruct +} diff --git a/shared-sqs/app/gosqs/change_message_visibility_test.go b/shared-sqs/app/gosqs/change_message_visibility_test.go new file mode 100644 index 0000000..2e9adac --- /dev/null +++ b/shared-sqs/app/gosqs/change_message_visibility_test.go @@ -0,0 +1,75 @@ +package gosqs + +import ( + "net/http" + "testing" + + "shared-sqs/app/test" + + "shared-sqs/app/fixtures" + "shared-sqs/app/models" + "github.com/stretchr/testify/assert" +) + +func TestChangeMessageVisibility_success(t *testing.T) { + // create a queue + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + }() + + q := &models.Queue{ + Name: "testing", + Messages: []models.SqsMessage{{ + MessageBody: "test1", + ReceiptHandle: "123", + }}, + } + models.SyncQueues.Queues["testing"] = q + + // The default value for the VisibilityTimeout is the zero value of time.Time + assert.Zero(t, q.Messages[0].VisibilityTimeout) + + _, r := test.GenerateRequestInfo("POST", "/", models.ChangeMessageVisibilityRequest{ + QueueUrl: "http://localhost:4100/queue/testing", + ReceiptHandle: "123", + VisibilityTimeout: 0, + }, true) + status, _ := ChangeMessageVisibilityV1(r) + assert.Equal(t, status, http.StatusOK) + + // Changing the message visibility increments the time.Time by N seconds + // from the current time. + // + // Given that the current time is relative between calling the endpoint and + // the time being set, we can't reliably assert an exact value. So assert + // that the time.Time value is no longer the default zero value. + assert.NotZero(t, q.Messages[0].VisibilityTimeout) + assert.NotZero(t, q.Messages[0].ReceiptTime) + assert.Equal(t, "", q.Messages[0].ReceiptHandle) + assert.Equal(t, 1, q.Messages[0].Retry) +} + +func TestChangeMessageVisibility_success_adds_to_existing_visibility_timeout(t *testing.T) { + // TODO +} + +func TestChangeMessageVisibility_success_transfers_to_dead_letter_queue(t *testing.T) { + // TODO +} + +func TestChangeMessageVisibility_request_transformer_error(t *testing.T) { + // TODO +} + +func TestChangeMessageVisibility_visibility_timeout_too_large(t *testing.T) { + // TODO +} + +func TestChangeMessageVisibility_missing_queue(t *testing.T) { + // TODO +} + +func TestChangeMessageVisibility_missing_message(t *testing.T) { + // TODO - mismatch receipt handle +} diff --git a/shared-sqs/app/gosqs/create_queue.go b/shared-sqs/app/gosqs/create_queue.go new file mode 100644 index 0000000..2c84faa --- /dev/null +++ b/shared-sqs/app/gosqs/create_queue.go @@ -0,0 +1,54 @@ +package gosqs + +import ( + "net/http" + "time" + + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + log "github.com/sirupsen/logrus" +) + +func CreateQueueV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewCreateQueueRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - CreateQueueV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + queueName := requestBody.QueueName + + queueUrl := "http://" + models.CurrentEnvironment.Host + ":" + models.CurrentEnvironment.Port + + "/" + models.CurrentEnvironment.AccountID + "/" + queueName + if models.CurrentEnvironment.Region != "" { + queueUrl = "http://" + models.CurrentEnvironment.Region + "." + models.CurrentEnvironment.Host + ":" + + models.CurrentEnvironment.Port + "/" + models.CurrentEnvironment.AccountID + "/" + queueName + } + queueArn := "arn:aws:sqs:" + models.CurrentEnvironment.Region + ":" + models.CurrentEnvironment.AccountID + ":" + queueName + + if _, ok := models.SyncQueues.Queues[queueName]; !ok { + log.Infof("Creating Queue: %s", queueName) + queue := &models.Queue{ + Name: queueName, + URL: queueUrl, + Arn: queueArn, + IsFIFO: utils.HasFIFOQueueName(queueName), + EnableDuplicates: models.CurrentEnvironment.EnableDuplicates, + Duplicates: make(map[string]time.Time), + } + if err := setQueueAttributesV1(queue, requestBody.Attributes); err != nil { + return utils.CreateErrorResponseV1(err.Error(), true) + } + models.SyncQueues.Lock() + models.SyncQueues.Queues[queueName] = queue + models.SyncQueues.Unlock() + } + + respStruct := models.CreateQueueResponse{ + Xmlns: models.BaseXmlns, + Result: models.CreateQueueResult{QueueUrl: queueUrl}, + Metadata: models.BaseResponseMetadata, + } + return http.StatusOK, respStruct +} diff --git a/shared-sqs/app/gosqs/create_queue_test.go b/shared-sqs/app/gosqs/create_queue_test.go new file mode 100644 index 0000000..0e47ec7 --- /dev/null +++ b/shared-sqs/app/gosqs/create_queue_test.go @@ -0,0 +1,254 @@ +package gosqs + +import ( + "fmt" + "net/http" + "testing" + "time" + + "shared-sqs/app/test" + + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/mitchellh/copystructure" + "github.com/stretchr/testify/assert" +) + +func TestCreateQueueV1_success(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.CreateQueueRequest) + *v = fixtures.CreateQueueRequest + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := CreateQueueV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, fixtures.CreateQueueResponse, response) + + actualQueue := models.SyncQueues.Queues[fixtures.QueueName] + assert.Equal(t, fixtures.FullyPopulatedQueue, actualQueue) +} + +func TestCreateQueueV1_success_with_redrive_policy(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + dupe, _ := copystructure.Copy(fixtures.CreateQueueRequest) + c, _ := dupe.(models.CreateQueueRequest) + c.Attributes.RedrivePolicy = models.RedrivePolicy{ + MaxReceiveCount: 100, + DeadLetterTargetArn: fmt.Sprintf("arn:aws:sqs:us-east-1:100010001000:%s", fixtures.DeadLetterQueueName), + } + + v := resultingStruct.(*models.CreateQueueRequest) + *v = c + return true + } + + dlq := &models.Queue{ + Name: fixtures.DeadLetterQueueName, + } + models.SyncQueues.Queues[fixtures.DeadLetterQueueName] = dlq + + expectedQueue := &models.Queue{ + Name: fixtures.QueueName, + URL: fmt.Sprintf("http://%s.%s:%s/%s/%s", + fixtures.LOCAL_ENVIRONMENT.Region, + fixtures.LOCAL_ENVIRONMENT.Host, + fixtures.LOCAL_ENVIRONMENT.Port, + fixtures.LOCAL_ENVIRONMENT.AccountID, + fixtures.QueueName, + ), + Arn: fmt.Sprintf("arn:aws:sqs:%s:%s:%s", + fixtures.LOCAL_ENVIRONMENT.Region, + fixtures.LOCAL_ENVIRONMENT.AccountID, + fixtures.QueueName, + ), + VisibilityTimeout: 5, + ReceiveMessageWaitTimeSeconds: 4, + DelaySeconds: 1, + MaximumMessageSize: 2, + MessageRetentionPeriod: 3, + DeadLetterQueue: dlq, + MaxReceiveCount: 100, + Duplicates: make(map[string]time.Time), + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := CreateQueueV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, fixtures.CreateQueueResponse, response) + + actualQueue := models.SyncQueues.Queues[fixtures.QueueName] + assert.Equal(t, expectedQueue, actualQueue) +} + +func TestCreateQueueV1_success_with_existing_queue(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.CreateQueueRequest) + *v = fixtures.CreateQueueRequest + return true + } + + q := &models.Queue{ + Name: fixtures.QueueName, + } + models.SyncQueues.Queues[fixtures.QueueName] = q + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := CreateQueueV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, fixtures.CreateQueueResponse, response) + + actualQueue := models.SyncQueues.Queues[fixtures.QueueName] + assert.Equal(t, q, actualQueue) +} + +func TestCreateQueueV1_success_with_no_request_attributes_falls_back_to_default(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + dupe, _ := copystructure.Copy(fixtures.CreateQueueRequest) + c, _ := dupe.(models.CreateQueueRequest) + c.Attributes = models.QueueAttributes{} + + v := resultingStruct.(*models.CreateQueueRequest) + *v = c + return true + } + + expectedQueue := &models.Queue{ + Name: fixtures.QueueName, + URL: fmt.Sprintf("http://%s.%s:%s/%s/%s", + fixtures.LOCAL_ENVIRONMENT.Region, + fixtures.LOCAL_ENVIRONMENT.Host, + fixtures.LOCAL_ENVIRONMENT.Port, + fixtures.LOCAL_ENVIRONMENT.AccountID, + fixtures.QueueName, + ), + Arn: fmt.Sprintf("arn:aws:sqs:%s:%s:%s", + fixtures.LOCAL_ENVIRONMENT.Region, + fixtures.LOCAL_ENVIRONMENT.AccountID, + fixtures.QueueName, + ), + VisibilityTimeout: 0, + ReceiveMessageWaitTimeSeconds: 0, + DelaySeconds: 0, + MaximumMessageSize: 0, + MessageRetentionPeriod: 0, + Duplicates: make(map[string]time.Time), + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := CreateQueueV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, fixtures.CreateQueueResponse, response) + + actualQueue := models.SyncQueues.Queues[fixtures.QueueName] + assert.Equal(t, expectedQueue, actualQueue) +} + +func TestCreateQueueV1_success_no_configured_region_for_queue_url(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + models.CurrentEnvironment.Region = "" + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + dupe, _ := copystructure.Copy(fixtures.CreateQueueRequest) + c, _ := dupe.(models.CreateQueueRequest) + c.Attributes = models.QueueAttributes{} + + v := resultingStruct.(*models.CreateQueueRequest) + *v = c + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := CreateQueueV1(r) + + assert.Equal(t, http.StatusOK, code) + + actualQueue := models.SyncQueues.Queues[fixtures.QueueName] + assert.Equal(t, + fmt.Sprintf("http://%s:%s/%s/%s", + fixtures.LOCAL_ENVIRONMENT.Host, + fixtures.LOCAL_ENVIRONMENT.Port, + fixtures.LOCAL_ENVIRONMENT.AccountID, + fixtures.QueueName, + ), + actualQueue.URL, + ) +} + +func TestCreateQueueV1_request_transformer_error(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := CreateQueueV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} + +func TestCreateQueueV1_invalid_dead_letter_queue_error(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + dupe, _ := copystructure.Copy(fixtures.CreateQueueRequest) + c, _ := dupe.(models.CreateQueueRequest) + c.Attributes.RedrivePolicy = models.RedrivePolicy{ + MaxReceiveCount: 100, + DeadLetterTargetArn: fmt.Sprintf("arn:aws:sqs:us-east-1:100010001000:%s", "garbage"), + } + + v := resultingStruct.(*models.CreateQueueRequest) + *v = c + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := CreateQueueV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} diff --git a/shared-sqs/app/gosqs/delete_message.go b/shared-sqs/app/gosqs/delete_message.go new file mode 100644 index 0000000..0f7080a --- /dev/null +++ b/shared-sqs/app/gosqs/delete_message.go @@ -0,0 +1,65 @@ +package gosqs + +import ( + "net/http" + "strings" + + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" +) + +func DeleteMessageV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewDeleteMessageRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - DeleteMessageV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + // Retrieve FormValues required + receiptHandle := requestBody.ReceiptHandle + + // Retrieve FormValues required + queueUrl := requestBody.QueueUrl + queueName := "" + if queueUrl == "" { + vars := mux.Vars(req) + queueName = vars["queueName"] + } else { + uriSegments := strings.Split(queueUrl, "/") + queueName = uriSegments[len(uriSegments)-1] + } + + log.Info("Deleting Message, Queue:", queueName, ", ReceiptHandle:", receiptHandle) + + // Find queue/message with the receipt handle and delete + models.SyncQueues.Lock() + defer models.SyncQueues.Unlock() + if _, ok := models.SyncQueues.Queues[queueName]; ok { + for i, msg := range models.SyncQueues.Queues[queueName].Messages { + if msg.ReceiptHandle == receiptHandle { + // Unlock messages for the group + log.Debugf("FIFO Queue %s unlocking group %s:", queueName, msg.GroupID) + models.SyncQueues.Queues[queueName].UnlockGroup(msg.GroupID) + //Delete message from Q + models.SyncQueues.Queues[queueName].Messages = append(models.SyncQueues.Queues[queueName].Messages[:i], models.SyncQueues.Queues[queueName].Messages[i+1:]...) + delete(models.SyncQueues.Queues[queueName].Duplicates, msg.DeduplicationID) + + // Create, encode/xml and send response + respStruct := models.DeleteMessageResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + return 200, &respStruct + } + } + log.Warning("Receipt Handle not found") + } else { + log.Warning("Queue not found") + } + + return utils.CreateErrorResponseV1("MessageDoesNotExist", true) +} diff --git a/shared-sqs/app/gosqs/delete_message_batch.go b/shared-sqs/app/gosqs/delete_message_batch.go new file mode 100644 index 0000000..01f1a23 --- /dev/null +++ b/shared-sqs/app/gosqs/delete_message_batch.go @@ -0,0 +1,118 @@ +package gosqs + +import ( + "net/http" + "strings" + + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" +) + +func DeleteMessageBatchV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewDeleteMessageBatchRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - DeleteMessageBatchV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + queueUrl := requestBody.QueueUrl + + queueName := "" + if queueUrl == "" { + vars := mux.Vars(req) + queueName = vars["queueName"] + } else { + uriSegments := strings.Split(queueUrl, "/") + queueName = uriSegments[len(uriSegments)-1] + } + + if _, ok := models.SyncQueues.Queues[queueName]; !ok { + return utils.CreateErrorResponseV1("QueueNotFound", true) + } + + if len(requestBody.Entries) == 0 { + return utils.CreateErrorResponseV1("EmptyBatchRequest", true) + } + + if len(requestBody.Entries) > 10 { + return utils.CreateErrorResponseV1("TooManyEntriesInBatchRequest", true) + } + + ids := map[string]bool{} + for _, v := range requestBody.Entries { + if _, found := ids[v.Id]; found { + return utils.CreateErrorResponseV1("BatchEntryIdsNotDistinct", true) + } + ids[v.Id] = true + } + + models.SyncQueues.Lock() + defer models.SyncQueues.Unlock() + + // create deleteMessageMap + deleteMessageMap := make(map[string]*deleteEntry) + for _, entry := range requestBody.Entries { + deleteMessageMap[entry.ReceiptHandle] = &deleteEntry{ + Id: entry.Id, + ReceiptHandle: entry.ReceiptHandle, + Deleted: false, + } + } + + deletedEntries := make([]models.DeleteMessageBatchResultEntry, 0) + // create a slice to hold messages that are not deleted + remainingMessages := make([]models.SqsMessage, 0, len(models.SyncQueues.Queues[queueName].Messages)) + + // delete message from queue + for _, message := range models.SyncQueues.Queues[queueName].Messages { + if deleteEntry, found := deleteMessageMap[message.ReceiptHandle]; found { + // Unlock messages for the group + log.Debugf("FIFO Queue %s unlocking group %s:", queueName, message.GroupID) + models.SyncQueues.Queues[queueName].UnlockGroup(message.GroupID) + delete(models.SyncQueues.Queues[queueName].Duplicates, message.DeduplicationID) + deleteEntry.Deleted = true + deletedEntries = append(deletedEntries, models.DeleteMessageBatchResultEntry{Id: deleteEntry.Id}) + } else { + remainingMessages = append(remainingMessages, message) + } + } + + // Update the queue with the remaining mesages + models.SyncQueues.Queues[queueName].Messages = remainingMessages + + // Process not found entries + notFoundEntries := make([]models.BatchResultErrorEntry, 0) + for _, deleteEntry := range deleteMessageMap { + if !deleteEntry.Deleted { + notFoundEntries = append(notFoundEntries, models.BatchResultErrorEntry{ + Code: "1", + Id: deleteEntry.Id, + Message: "Message not found", + SenderFault: true, + }) + } + } + + respStruct := models.DeleteMessageBatchResponse{ + Xmlns: models.BaseXmlns, + Result: models.DeleteMessageBatchResult{ + Successful: deletedEntries, + Failed: notFoundEntries, + }, + Metadata: models.BaseResponseMetadata, + } + + return http.StatusOK, respStruct + +} + +type deleteEntry struct { + Id string + ReceiptHandle string + Error string + Deleted bool +} diff --git a/shared-sqs/app/gosqs/delete_message_batch_test.go b/shared-sqs/app/gosqs/delete_message_batch_test.go new file mode 100644 index 0000000..87ff69c --- /dev/null +++ b/shared-sqs/app/gosqs/delete_message_batch_test.go @@ -0,0 +1,329 @@ +package gosqs + +import ( + "fmt" + "net/http" + "testing" + + "shared-sqs/app/conf" + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/test" + "shared-sqs/app/utils" + "github.com/stretchr/testify/assert" +) + +func TestDeleteMessageBatchV1_success_all_message(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + q := &models.Queue{ + Name: "testing", + Messages: []models.SqsMessage{ + { + MessageBody: "test%20message%20body%201", + ReceiptHandle: "test1", + }, + { + MessageBody: "test%20message%20body%202", + ReceiptHandle: "test2", + }, + { + MessageBody: "test%20message%20body%203", + ReceiptHandle: "test3", + }, + }, + } + models.SyncQueues.Queues["testing"] = q + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.DeleteMessageBatchRequest) + *v = models.DeleteMessageBatchRequest{ + Entries: []models.DeleteMessageBatchRequestEntry{ + { + Id: "delete-test-1", + ReceiptHandle: "test1", + }, + { + Id: "delete-test-2", + ReceiptHandle: "test2", + }, + { + Id: "delete-test-3", + ReceiptHandle: "test3", + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "testing"), + } + return true + } + _, request2 := test.GenerateRequestInfo( + "POST", + "/", + nil, + true) + + status, response2 := DeleteMessageBatchV1(request2) + deleteMessageBatchResponse := response2.(models.DeleteMessageBatchResponse) + assert.Equal(t, status, http.StatusOK) + assert.Equal(t, "delete-test-1", deleteMessageBatchResponse.Result.Successful[0].Id) + assert.Equal(t, "delete-test-2", deleteMessageBatchResponse.Result.Successful[1].Id) + assert.Equal(t, "delete-test-3", deleteMessageBatchResponse.Result.Successful[2].Id) + assert.Empty(t, deleteMessageBatchResponse.Result.Failed) + assert.Empty(t, models.SyncQueues.Queues["testing"].Messages) +} +func TestDeleteMessageBatchV1_success_not_found_message(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + q := &models.Queue{ + Name: "testing", + Messages: []models.SqsMessage{ + { + MessageBody: "test%20message%20body%201", + ReceiptHandle: "test1", + }, + { + MessageBody: "test%20message%20body%203", + ReceiptHandle: "test3", + }, + }, + } + models.SyncQueues.Queues["testing"] = q + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.DeleteMessageBatchRequest) + *v = models.DeleteMessageBatchRequest{ + Entries: []models.DeleteMessageBatchRequestEntry{ + { + Id: "delete-test-1", + ReceiptHandle: "test1", + }, + { + Id: "delete-test-2", + ReceiptHandle: "test2", + }, + { + Id: "delete-test-3", + ReceiptHandle: "test3", + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "testing"), + } + return true + } + _, request := test.GenerateRequestInfo( + "POST", + "/", + nil, + true) + + status, response := DeleteMessageBatchV1(request) + deleteMessageBatchResponse := response.(models.DeleteMessageBatchResponse) + assert.Equal(t, status, http.StatusOK) + assert.Equal(t, "delete-test-1", deleteMessageBatchResponse.Result.Successful[0].Id) + assert.Equal(t, "delete-test-3", deleteMessageBatchResponse.Result.Successful[1].Id) + assert.Equal(t, "1", deleteMessageBatchResponse.Result.Failed[0].Code) + assert.Equal(t, "delete-test-2", deleteMessageBatchResponse.Result.Failed[0].Id) + assert.Equal(t, "Message not found", deleteMessageBatchResponse.Result.Failed[0].Message) + assert.True(t, deleteMessageBatchResponse.Result.Failed[0].SenderFault) + assert.Empty(t, models.SyncQueues.Queues["testing"].Messages) +} + +func TestDeleteMessageBatchV1_error_not_found_queue(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.DeleteMessageBatchRequest) + *v = models.DeleteMessageBatchRequest{ + Entries: []models.DeleteMessageBatchRequestEntry{ + { + Id: "delete-test-1", + ReceiptHandle: "test1", + }, + { + Id: "delete-test-2", + ReceiptHandle: "test2", + }, + { + Id: "delete-test-3", + ReceiptHandle: "test3", + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "not-exist-queue"), + } + return true + } + _, r := test.GenerateRequestInfo( + "POST", + "/", + nil, + true) + + status, _ := DeleteMessageBatchV1(r) + assert.Equal(t, status, http.StatusBadRequest) + +} + +func TestDeleteMessageBatchV1_error_no_entry(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.DeleteMessageBatchRequest) + *v = models.DeleteMessageBatchRequest{ + Entries: make([]models.DeleteMessageBatchRequestEntry, 0), + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + return true + } + _, r := test.GenerateRequestInfo( + "POST", + "/", + nil, + true) + + status, _ := DeleteMessageBatchV1(r) + assert.Equal(t, status, http.StatusBadRequest) +} + +func TestDeleteMessageBatchV1_error_too_many_entries(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.DeleteMessageBatchRequest) + *v = models.DeleteMessageBatchRequest{ + Entries: []models.DeleteMessageBatchRequestEntry{ + { + Id: "test-1", + ReceiptHandle: "test-1", + }, + { + Id: "test-2", + ReceiptHandle: "test-2", + }, + { + Id: "test-3", + ReceiptHandle: "test-3", + }, + { + Id: "test-4", + ReceiptHandle: "test-4", + }, + { + Id: "test-5", + ReceiptHandle: "test-5", + }, + { + Id: "test-6", + ReceiptHandle: "test-6", + }, + { + Id: "test-7", + ReceiptHandle: "test-7", + }, + { + Id: "test-8", + ReceiptHandle: "test-8", + }, + { + Id: "test-9", + ReceiptHandle: "test-9", + }, + { + Id: "test-10", + ReceiptHandle: "test-10", + }, + { + Id: "test-11", + ReceiptHandle: "test-11", + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + return true + } + _, r := test.GenerateRequestInfo( + "POST", + "/", + nil, + true) + + status, _ := DeleteMessageBatchV1(r) + assert.Equal(t, status, http.StatusBadRequest) +} + +func TestDeleteMessageBatchV1_Error_IdNotDistinct(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.DeleteMessageBatchRequest) + *v = models.DeleteMessageBatchRequest{ + Entries: []models.DeleteMessageBatchRequestEntry{ + { + Id: "delete-test-1", + ReceiptHandle: "test1", + }, + { + Id: "delete-test-1", + ReceiptHandle: "test2", + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + return true + } + _, r := test.GenerateRequestInfo( + "POST", + "/", + nil, + true) + + status, _ := DeleteMessageBatchV1(r) + assert.Equal(t, http.StatusBadRequest, status) +} + +func TestDeleteMessageBatchV1_Error_transformer(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, _ := DeleteMessageBatchV1(r) + + assert.Equal(t, http.StatusBadRequest, status) + +} diff --git a/shared-sqs/app/gosqs/delete_message_test.go b/shared-sqs/app/gosqs/delete_message_test.go new file mode 100644 index 0000000..0722f72 --- /dev/null +++ b/shared-sqs/app/gosqs/delete_message_test.go @@ -0,0 +1,38 @@ +package gosqs + +import ( + "net/http" + "testing" + + "shared-sqs/app/test" + + "shared-sqs/app/fixtures" + "shared-sqs/app/models" + "github.com/stretchr/testify/assert" +) + +func TestDeleteMessage(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + }() + + q := &models.Queue{ + Name: "testing", + Messages: []models.SqsMessage{{ + MessageBody: "test1", + ReceiptHandle: "123", + }}, + } + + models.SyncQueues.Queues["testing"] = q + + _, r := test.GenerateRequestInfo("POST", "/", models.DeleteMessageRequest{ + QueueUrl: "http://localhost:4100/queue/testing", + ReceiptHandle: "123", + }, true) + status, _ := DeleteMessageV1(r) + + assert.Equal(t, status, http.StatusOK) + assert.Empty(t, q.Messages) +} diff --git a/shared-sqs/app/gosqs/delete_queue.go b/shared-sqs/app/gosqs/delete_queue.go new file mode 100644 index 0000000..6f21d80 --- /dev/null +++ b/shared-sqs/app/gosqs/delete_queue.go @@ -0,0 +1,37 @@ +package gosqs + +import ( + "net/http" + "strings" + + "shared-sqs/app/interfaces" + + "shared-sqs/app/models" + "shared-sqs/app/utils" + + log "github.com/sirupsen/logrus" +) + +func DeleteQueueV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewDeleteQueueRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - DeleteQueueV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + uriSegments := strings.Split(requestBody.QueueUrl, "/") + queueName := uriSegments[len(uriSegments)-1] + + log.Infof("Deleting Queue: %s", queueName) + + models.SyncQueues.Lock() + delete(models.SyncQueues.Queues, queueName) + models.SyncQueues.Unlock() + + respStruct := models.DeleteQueueResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + return http.StatusOK, respStruct +} diff --git a/shared-sqs/app/gosqs/delete_queue_test.go b/shared-sqs/app/gosqs/delete_queue_test.go new file mode 100644 index 0000000..fc279d6 --- /dev/null +++ b/shared-sqs/app/gosqs/delete_queue_test.go @@ -0,0 +1,91 @@ +package gosqs + +import ( + "fmt" + "net/http" + "testing" + + "shared-sqs/app/test" + + "github.com/stretchr/testify/assert" + + "shared-sqs/app/conf" + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" +) + +func TestDeleteQueueV1_success(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.DeleteQueueRequest) + *v = models.DeleteQueueRequest{ + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + return true + } + + expectedResponse := models.DeleteQueueResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := DeleteQueueV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, expectedResponse, response) + + _, ok := models.SyncQueues.Queues["unit-queue1"] + assert.False(t, ok) +} + +func TestDeleteQueueV1_success_unknown_queue(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.DeleteQueueRequest) + *v = models.DeleteQueueRequest{ + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unknown-queue1"), + } + return true + } + + expectedResponse := models.DeleteQueueResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := DeleteQueueV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, expectedResponse, response) +} + +func TestDeleteQueueV1_error_invalid_request(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := DeleteQueueV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} diff --git a/shared-sqs/app/gosqs/get_queue_attributes.go b/shared-sqs/app/gosqs/get_queue_attributes.go new file mode 100644 index 0000000..2e0d95d --- /dev/null +++ b/shared-sqs/app/gosqs/get_queue_attributes.go @@ -0,0 +1,135 @@ +package gosqs + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/mitchellh/copystructure" + + "shared-sqs/app/interfaces" + + log "github.com/sirupsen/logrus" +) + +func GetQueueAttributesV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewGetQueueAttributesRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - GetQueueAttributesV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + if requestBody.QueueUrl == "" { + log.Error("Missing QueueUrl - GetQueueAttributesV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + requestedAttributes := func() map[string]bool { + attrs := map[string]bool{} + if len(requestBody.AttributeNames) == 0 { + return map[string]bool{"All": true} + } + for _, attr := range requestBody.AttributeNames { + if "All" == attr { + return map[string]bool{"All": true} + } + attrs[attr] = true + } + return attrs + }() + + dupe, _ := copystructure.Copy(models.AvailableQueueAttributes) + includedAttributes, _ := dupe.(map[string]bool) + _, ok = requestedAttributes["All"] + if !ok { + for attr, _ := range includedAttributes { + _, ok := requestedAttributes[attr] + if !ok { + delete(includedAttributes, attr) + } + } + } + + uriSegments := strings.Split(requestBody.QueueUrl, "/") + queueName := uriSegments[len(uriSegments)-1] + + log.Infof("Get Queue QueueAttributes: %s", queueName) + queueAttributes := make([]models.Attribute, 0, 0) + + models.SyncQueues.RLock() + defer models.SyncQueues.RUnlock() + queue, ok := models.SyncQueues.Queues[queueName] + if !ok { + log.Errorf("Get Queue URL: %s queue does not exist!!!", queueName) + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + if _, ok := includedAttributes["DelaySeconds"]; ok { + attr := models.Attribute{Name: "DelaySeconds", Value: strconv.Itoa(queue.DelaySeconds)} + queueAttributes = append(queueAttributes, attr) + } + if _, ok := includedAttributes["MaximumMessageSize"]; ok { + attr := models.Attribute{Name: "MaximumMessageSize", Value: strconv.Itoa(queue.MaximumMessageSize)} + queueAttributes = append(queueAttributes, attr) + } + if _, ok := includedAttributes["MessageRetentionPeriod"]; ok { + attr := models.Attribute{Name: "MessageRetentionPeriod", Value: strconv.Itoa(queue.MessageRetentionPeriod)} + queueAttributes = append(queueAttributes, attr) + } + if _, ok := includedAttributes["ReceiveMessageWaitTimeSeconds"]; ok { + attr := models.Attribute{Name: "ReceiveMessageWaitTimeSeconds", Value: strconv.Itoa(queue.ReceiveMessageWaitTimeSeconds)} + queueAttributes = append(queueAttributes, attr) + } + if _, ok := includedAttributes["VisibilityTimeout"]; ok { + attr := models.Attribute{Name: "VisibilityTimeout", Value: strconv.Itoa(queue.VisibilityTimeout)} + queueAttributes = append(queueAttributes, attr) + } + if _, ok := includedAttributes["ApproximateNumberOfMessages"]; ok { + attr := models.Attribute{Name: "ApproximateNumberOfMessages", Value: strconv.Itoa(len(queue.Messages))} + queueAttributes = append(queueAttributes, attr) + } + // TODO - implement + //if _, ok := includedAttributes["ApproximateNumberOfMessagesDelayed"]; ok { + // attr := models.Attribute{Name: "ApproximateNumberOfMessagesDelayed", Value: strconv.Itoa(len(queue.Messages))} + // queueAttributes = append(queueAttributes, attr) + //} + if _, ok := includedAttributes["ApproximateNumberOfMessagesNotVisible"]; ok { + attr := models.Attribute{Name: "ApproximateNumberOfMessagesNotVisible", Value: strconv.Itoa(numberOfHiddenMessagesInQueue(*queue))} + queueAttributes = append(queueAttributes, attr) + } + if _, ok := includedAttributes["CreatedTimestamp"]; ok { + attr := models.Attribute{Name: "CreatedTimestamp", Value: "0000000000"} + queueAttributes = append(queueAttributes, attr) + } + if _, ok := includedAttributes["LastModifiedTimestamp"]; ok { + attr := models.Attribute{Name: "LastModifiedTimestamp", Value: "0000000000"} + queueAttributes = append(queueAttributes, attr) + } + if _, ok := includedAttributes["QueueArn"]; ok { + attr := models.Attribute{Name: "QueueArn", Value: queue.Arn} + queueAttributes = append(queueAttributes, attr) + } + // TODO - implement + //if _, ok := includedAttributes["Policy"]; ok { + // attr := models.Attribute{Name: "Policy", Value: ""} + // queueAttributes = append(queueAttributes, attr) + //} + //if _, ok := includedAttributes["RedriveAllowPolicy"]; ok { + // attr := models.Attribute{Name: "RedriveAllowPolicy", Value: ""} + // queueAttributes = append(queueAttributes, attr) + //} + if _, ok := includedAttributes["RedrivePolicy"]; ok && queue.DeadLetterQueue != nil { + attr := models.Attribute{Name: "RedrivePolicy", Value: fmt.Sprintf(`{"maxReceiveCount":"%d", "deadLetterTargetArn":"%s"}`, queue.MaxReceiveCount, queue.DeadLetterQueue.Arn)} + queueAttributes = append(queueAttributes, attr) + } + + respStruct := models.GetQueueAttributesResponse{ + Xmlns: models.BaseXmlns, + Result: models.GetQueueAttributesResult{Attrs: queueAttributes}, + Metadata: models.BaseResponseMetadata, + } + return http.StatusOK, respStruct +} diff --git a/shared-sqs/app/gosqs/get_queue_attributes_test.go b/shared-sqs/app/gosqs/get_queue_attributes_test.go new file mode 100644 index 0000000..885dc3c --- /dev/null +++ b/shared-sqs/app/gosqs/get_queue_attributes_test.go @@ -0,0 +1,180 @@ +package gosqs + +import ( + "fmt" + "net/http" + "testing" + + "shared-sqs/app/test" + + "github.com/mitchellh/copystructure" + + "shared-sqs/app/conf" + + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/stretchr/testify/assert" +) + +func TestGetQueueAttributesV1_success_all(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.GetQueueAttributesRequest) + *v = fixtures.GetQueueAttributesRequest + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := GetQueueAttributesV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, fixtures.GetQueueAttributesResponse, response) +} + +func TestGetQueueAttributesV1_success_no_request_attrs_returns_all(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.GetQueueAttributesRequest) + *v = models.GetQueueAttributesRequest{ + QueueUrl: "unit-queue1", + } + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := GetQueueAttributesV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, fixtures.GetQueueAttributesResponse, response) +} + +func TestGetQueueAttributesV1_success_all_with_redrive_queue(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.GetQueueAttributesRequest) + *v = models.GetQueueAttributesRequest{ + QueueUrl: "unit-queue2", + AttributeNames: []string{"All"}, + } + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := GetQueueAttributesV1(r) + + dupe, _ := copystructure.Copy(fixtures.GetQueueAttributesResponse) + expectedResponse, _ := dupe.(models.GetQueueAttributesResponse) + expectedResponse.Result.Attrs[9].Value = fmt.Sprintf("%s:%s", fixtures.BASE_SQS_ARN, "unit-queue2") + expectedResponse.Result.Attrs = append(expectedResponse.Result.Attrs, + models.Attribute{ + Name: "RedrivePolicy", + Value: fmt.Sprintf(`{"maxReceiveCount":"1", "deadLetterTargetArn":"%s:%s"}`, fixtures.BASE_SQS_ARN, "dead-letter-queue1"), + }, + ) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, expectedResponse, response) +} + +func TestGetQueueAttributesV1_success_specific_fields(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.GetQueueAttributesRequest) + *v = models.GetQueueAttributesRequest{ + QueueUrl: fmt.Sprintf("%s/unit-queue1", fixtures.BASE_URL), + AttributeNames: []string{"DelaySeconds"}, + } + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := GetQueueAttributesV1(r) + + expectedResponse := models.GetQueueAttributesResponse{ + Xmlns: models.BaseXmlns, + Result: models.GetQueueAttributesResult{Attrs: []models.Attribute{ + models.Attribute{ + Name: "DelaySeconds", + Value: "0", + }, + }}, + Metadata: models.BaseResponseMetadata, + } + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, expectedResponse, response) +} + +func TestGetQueueAttributesV1_request_transformer_error(t *testing.T) { + defer func() { + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := GetQueueAttributesV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} + +func TestGetQueueAttributesV1_missing_queue_url_in_request_returns_error(t *testing.T) { + defer func() { + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.GetQueueAttributesRequest) + *v = models.GetQueueAttributesRequest{ + QueueUrl: "", + AttributeNames: []string{}, + } + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := GetQueueAttributesV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} + +func TestGetQueueAttributesV1_missing_queue_returns_error(t *testing.T) { + defer func() { + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.GetQueueAttributesRequest) + *v = fixtures.GetQueueAttributesRequest + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := GetQueueAttributesV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} diff --git a/shared-sqs/app/gosqs/get_queue_url.go b/shared-sqs/app/gosqs/get_queue_url.go new file mode 100644 index 0000000..258724b --- /dev/null +++ b/shared-sqs/app/gosqs/get_queue_url.go @@ -0,0 +1,36 @@ +package gosqs + +import ( + "net/http" + + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + log "github.com/sirupsen/logrus" +) + +func GetQueueUrlV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewGetQueueUrlRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - GetQueueUrlV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + queueName := requestBody.QueueName + if _, ok := models.SyncQueues.Queues[queueName]; !ok { + log.Error("Get Queue URL:", queueName, ", queue does not exist!!!") + return utils.CreateErrorResponseV1("QueueNotFound", true) + } + + queue := models.SyncQueues.Queues[queueName] + log.Debug("Get Queue URL:", queue.Name) + + result := models.GetQueueUrlResult{QueueUrl: queue.URL} + respStruct := models.GetQueueUrlResponse{ + Xmlns: models.BaseXmlns, + Result: result, + Metadata: models.BaseResponseMetadata, + } + return http.StatusOK, respStruct +} diff --git a/shared-sqs/app/gosqs/get_queue_url_test.go b/shared-sqs/app/gosqs/get_queue_url_test.go new file mode 100644 index 0000000..d3c5615 --- /dev/null +++ b/shared-sqs/app/gosqs/get_queue_url_test.go @@ -0,0 +1,104 @@ +package gosqs + +import ( + "fmt" + "net/http" + "testing" + + "shared-sqs/app/test" + + "shared-sqs/app/conf" + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/stretchr/testify/assert" +) + +func TestGetQueueUrlV1_success(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.GetQueueUrlRequest) + *v = models.GetQueueUrlRequest{ + QueueName: "unit-queue1", + QueueOwnerAWSAccountId: "fugafuga", + } + return true + } + + _, r := test.GenerateRequestInfo( + "POST", + "/", + nil, + true) + code, response := GetQueueUrlV1(r) + + get_queue_url_response := response.(models.GetQueueUrlResponse) + + assert.Equal(t, http.StatusOK, code) + assert.Contains(t, get_queue_url_response.Result.QueueUrl, fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1")) + +} + +func TestGetQueueUrlV1_error_no_queue(t *testing.T) { + + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.GetQueueUrlRequest) + *v = models.GetQueueUrlRequest{ + QueueName: "not-exist-unit-queue1", + QueueOwnerAWSAccountId: "fugafuga", + } + return true + } + + _, r := test.GenerateRequestInfo( + "POST", + "/", + nil, + true) + code, response := GetQueueUrlV1(r) + + expected := models.ErrorResult{ + Type: "Not Found", + Code: "AWS.SimpleQueueService.NonExistentQueue", + Message: "The specified queue does not exist for this wsdl version.", + } + + assert.Equal(t, http.StatusBadRequest, code) + assert.Equal(t, response.GetResult().(models.ErrorResult), expected) +} + +func TestGetQueueUrlV1_error_request_transformer(t *testing.T) { + + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo( + "POST", + "/", + nil, + true) + code, _ := GetQueueUrlV1(r) + assert.Equal(t, http.StatusBadRequest, code) +} diff --git a/shared-sqs/app/gosqs/gosqs.go b/shared-sqs/app/gosqs/gosqs.go new file mode 100644 index 0000000..6a45977 --- /dev/null +++ b/shared-sqs/app/gosqs/gosqs.go @@ -0,0 +1,82 @@ +package gosqs + +import ( + "net/url" + "time" + + "shared-sqs/app/models" + + log "github.com/sirupsen/logrus" +) + +func init() { + models.SyncQueues.Queues = make(map[string]*models.Queue) +} + +func PeriodicTasks(d time.Duration, quit chan bool) { + ticker := time.NewTicker(d) + for { + select { + case <-ticker.C: + models.SyncQueues.Lock() + for qName := range models.SyncQueues.Queues { + queue := models.SyncQueues.Queues[qName] + + // Reset deduplication period + for dedupId, startTime := range queue.Duplicates { + if time.Now().After(startTime.Add(models.DeduplicationPeriod)) { + log.Debugf("deduplication period for message with deduplicationId [%s] expired", dedupId) + delete(queue.Duplicates, dedupId) + } + } + + log.Debugf("Queue [%s] length [%d]", queue.Name, len(queue.Messages)) + for i := 0; i < len(queue.Messages); i++ { + msg := &queue.Messages[i] + + if msg.ReceiptHandle != "" { + if msg.VisibilityTimeout.Before(time.Now()) { + log.Debugf("Making message visible again %s", msg.ReceiptHandle) + queue.UnlockGroup(msg.GroupID) + msg.ReceiptHandle = "" + msg.ReceiptTime = time.Now().UTC() + msg.Retry++ + if queue.MaxReceiveCount > 0 && + queue.DeadLetterQueue != nil && + msg.Retry >= queue.MaxReceiveCount { + queue.DeadLetterQueue.Messages = append(queue.DeadLetterQueue.Messages, *msg) + queue.Messages = append(queue.Messages[:i], queue.Messages[i+1:]...) + i-- + } + } + } + } + } + models.SyncQueues.Unlock() + case <-quit: + ticker.Stop() + return + } + } +} + +func numberOfHiddenMessagesInQueue(queue models.Queue) int { + num := 0 + for _, m := range queue.Messages { + if m.ReceiptHandle != "" || m.DelaySecs > 0 && time.Now().Before(m.SentTime.Add(time.Duration(m.DelaySecs)*time.Second)) { + num++ + } + } + return num +} + +func getQueueFromPath(formVal string, theUrl string) string { + if formVal != "" { + return formVal + } + u, err := url.Parse(theUrl) + if err != nil { + return "" + } + return u.Path +} diff --git a/shared-sqs/app/gosqs/gosqs_test.go b/shared-sqs/app/gosqs/gosqs_test.go new file mode 100644 index 0000000..4256703 --- /dev/null +++ b/shared-sqs/app/gosqs/gosqs_test.go @@ -0,0 +1,668 @@ +package gosqs + +import ( + "fmt" + "net/http" + "net/url" + "sync" + "testing" + "time" + + "shared-sqs/app/conf" + + "shared-sqs/app/fixtures" + + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/stretchr/testify/assert" +) + +// TODO - Admiral-Piett these are better but still screwy. It's easy to have race conditions in here, so +// we have to name all the queues uniquely and leave them around so we're not resetting ourselves. +// Stupid. Handle the global memory issues and this can be easily resolved. +func Test_PeriodicTasks_deletes_deduplication_period_upon_expiration(t *testing.T) { + models.DeduplicationPeriod = 20 * time.Millisecond + quit := make(chan bool) + defer func() { + models.ResetApp() + quit <- true + models.DeduplicationPeriod = 5 * time.Minute + }() + + qName := "gosqs-dedupe-queue1" + mainQueue := &models.Queue{ + Name: qName, + URL: fmt.Sprintf("%s/%s", fixtures.BASE_URL, qName), + Arn: fmt.Sprintf("%s:%s", fixtures.BASE_SQS_ARN, qName), + Duplicates: map[string]time.Time{ + "12345": time.Now(), + }, + } + models.SyncQueues.Lock() + models.SyncQueues.Queues[qName] = mainQueue + models.SyncQueues.Unlock() + + go PeriodicTasks(10*time.Millisecond, quit) + + assertions := func() bool { + models.SyncQueues.Lock() + defer models.SyncQueues.Unlock() + + ok := 0 == len(mainQueue.Duplicates) + if !ok { + return false + } + return true + } + assert.Eventually(t, assertions, 10*time.Second, 10*time.Millisecond) +} + +func Test_PeriodicTasks_VisibilityTimeout_expires(t *testing.T) { + quit := make(chan bool) + defer func() { + models.ResetApp() + quit <- true + }() + qName := "gosqs-visibility-queue1" + mainQueue := &models.Queue{ + Name: qName, + URL: fmt.Sprintf("%s/%s", fixtures.BASE_URL, qName), + Arn: fmt.Sprintf("%s:%s", fixtures.BASE_SQS_ARN, qName), + } + mainQueue.Messages = append(mainQueue.Messages, models.SqsMessage{ + MessageBody: "1", + ReceiptHandle: "12345", + VisibilityTimeout: time.Now().Add(30 * time.Millisecond), + }) + + models.SyncQueues.Lock() + models.SyncQueues.Queues[qName] = mainQueue + models.SyncQueues.Unlock() + + go PeriodicTasks(10*time.Millisecond, quit) + + assertions := func() bool { + models.SyncQueues.Lock() + defer models.SyncQueues.Unlock() + + ok := !mainQueue.Messages[0].ReceiptTime.IsZero() + if !ok { + return false + } + ok = "1" == mainQueue.Messages[0].MessageBody + if !ok { + return false + } + ok = "" == mainQueue.Messages[0].ReceiptHandle + if !ok { + return false + } + ok = 1 == mainQueue.Messages[0].Retry + if !ok { + return false + } + return true + } + assert.Eventually(t, assertions, 10*time.Second, 10*time.Millisecond) +} + +func Test_PeriodicTasks_moves_single_message_to_dead_letter_queue_upon_passing_receive_count(t *testing.T) { + quit := make(chan bool) + defer func() { + models.ResetApp() + quit <- true + }() + + qName := "gosqs-main-queue1" + dlqName := "gosqs-dead-letter-queue1" + dlqQueue := &models.Queue{ + Arn: fmt.Sprintf("%s/%s", fixtures.BASE_SQS_ARN, dlqName), + Name: dlqName, + URL: fmt.Sprintf("%s/%s", fixtures.BASE_URL, dlqName), + } + mainQueue := &models.Queue{ + Arn: fmt.Sprintf("%s/%s", fixtures.BASE_SQS_ARN, qName), + DeadLetterQueue: dlqQueue, + MaxReceiveCount: 1, + Name: qName, + URL: fmt.Sprintf("%s/%s", fixtures.BASE_URL, qName), + } + + go PeriodicTasks(10*time.Millisecond, quit) + + models.SyncQueues.Lock() + mainQueue.Messages = append(mainQueue.Messages, models.SqsMessage{ + MessageBody: "1", + Retry: 100, + ReceiptHandle: "12345", + VisibilityTimeout: time.Now().Add(10 * time.Millisecond), + }) + models.SyncQueues.Queues[qName] = mainQueue + models.SyncQueues.Queues[dlqName] = dlqQueue + models.SyncQueues.Unlock() + + assertions := func() bool { + models.SyncQueues.Lock() + defer models.SyncQueues.Unlock() + + ok := len(dlqQueue.Messages) == 1 + if !ok { + return false + } + ok = "1" == dlqQueue.Messages[0].MessageBody + if !ok { + return false + } + return true + } + assert.Eventually(t, assertions, 10*time.Second, 10*time.Millisecond) +} + +func Test_PeriodicTasks_moves_multiple_messages_to_dead_letter_queue_upon_passing_receive_count(t *testing.T) { + quit := make(chan bool) + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + quit <- true + }() + + mainQueue := models.SyncQueues.Queues["unit-queue2"] + dlqQueue := models.SyncQueues.Queues["dead-letter-queue1"] + + assert.Len(t, dlqQueue.Messages, 0) + + go PeriodicTasks(10*time.Millisecond, quit) + + models.SyncQueues.Lock() + mainQueue.Messages = append(mainQueue.Messages, models.SqsMessage{ + MessageBody: "1", + Retry: 100, + ReceiptHandle: "12345", + }) + mainQueue.Messages = append(mainQueue.Messages, models.SqsMessage{ + MessageBody: "2", + Retry: 100, + ReceiptHandle: "23456", + }) + models.SyncQueues.Unlock() + + assertions := func() bool { + models.SyncQueues.Lock() + defer models.SyncQueues.Unlock() + + ok := len(dlqQueue.Messages) == 2 + if !ok { + return false + } + ok = "1" == dlqQueue.Messages[0].MessageBody + if !ok { + return false + } + ok = "2" == dlqQueue.Messages[1].MessageBody + if !ok { + return false + } + return true + } + assert.Eventually(t, assertions, 10*time.Second, 10*time.Millisecond) +} + +// TODO - I think all these below belong in handler tests, not in here. Double check the relevant +// handlers for coverage and delete. +func TestSendingAndReceivingFromFIFOQueueReturnsSameMessageOnError(t *testing.T) { + done := make(chan bool) + go PeriodicTasks(1*time.Second, done) + + // create a queue + req, err := http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "requeue-reset.fifo") + form.Add("Attribute.1.Name", "VisibilityTimeout") + form.Add("Attribute.1.Value", "2") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ := CreateQueueV1(req) + assert.Equal(t, status, http.StatusOK) + + // send a message + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form = url.Values{} + form.Add("Action", "SendMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/requeue-reset.fifo") + form.Add("MessageBody", "1") + form.Add("MessageGroupId", "GROUP-X") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = SendMessageV1(req) + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", + status, http.StatusOK) + } + + // send a message + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form = url.Values{} + form.Add("Action", "SendMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/requeue-reset.fifo") + form.Add("MessageBody", "2") + form.Add("MessageGroupId", "GROUP-X") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = SendMessageV1(req) + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", + status, http.StatusOK) + } + + // receive message + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form = url.Values{} + form.Add("Action", "ReceiveMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/requeue-reset.fifo") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, resp := ReceiveMessageV1(req) + assert.Equal(t, status, http.StatusOK) + + result := resp.GetResult().(models.ReceiveMessageResult) + receiptHandleFirst := result.Messages[0].ReceiptHandle + if string(result.Messages[0].Body) != "1" { + t.Fatalf("should have received body 1: %s", err) + } + + // try to receive another message and we should get none + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form = url.Values{} + form.Add("Action", "ReceiveMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/requeue-reset.fifo") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = ReceiveMessageV1(req) + assert.Equal(t, status, http.StatusOK) + + if len(models.SyncQueues.Queues["requeue-reset.fifo"].FIFOMessages) != 1 { + t.Fatal("there should be only 1 group locked") + } + + if models.SyncQueues.Queues["requeue-reset.fifo"].FIFOMessages["GROUP-X"] != 0 { + t.Fatal("there should be GROUP-X locked") + } + + // remove message + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form = url.Values{} + form.Add("Action", "DeleteMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/requeue-reset.fifo") + form.Add("ReceiptHandle", receiptHandleFirst) + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = DeleteMessageV1(req) + assert.Equal(t, status, http.StatusOK) + + if len(models.SyncQueues.Queues["requeue-reset.fifo"].Messages) != 1 { + t.Fatal("there should be only 1 message in queue") + } + + // receive message - loop until visibility timeouts + for { + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form = url.Values{} + form.Add("Action", "ReceiveMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/requeue-reset.fifo") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, resp := ReceiveMessageV1(req) + assert.Equal(t, status, http.StatusOK) + + result := resp.GetResult().(models.ReceiveMessageResult) + if len(result.Messages) == 0 { + continue + } + + if string(result.Messages[0].Body) != "2" { + t.Fatalf("should have received body 2: %s", err) + } + break + } + + done <- true +} + +func TestSendMessage_POST_DuplicatationNotAppliedToStandardQueue(t *testing.T) { + done := make(chan bool) + go PeriodicTasks(1*time.Second, done) + + // create a queue + req, err := http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "stantdard-testing") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ := CreateQueueV1(req) + + assert.Equal(t, status, http.StatusOK) + + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form = url.Values{} + form.Add("Action", "SendMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/stantdard-testing") + form.Add("MessageBody", "Test1") + form.Add("MessageDeduplicationId", "123") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = SendMessageV1(req) + + // Check the status code is what we expect. + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", + status, http.StatusOK) + } + if len(models.SyncQueues.Queues["stantdard-testing"].Messages) == 0 { + t.Fatal("there should be 1 message in queue") + } + + form = url.Values{} + form.Add("Action", "SendMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/stantdard-testing") + form.Add("MessageBody", "Test2") + form.Add("MessageDeduplicationId", "123") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = SendMessageV1(req) + + // Check the status code is what we expect. + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", + status, http.StatusOK) + } + if len(models.SyncQueues.Queues["stantdard-testing"].Messages) == 1 { + t.Fatal("there should be 2 messages in queue") + } + done <- true +} + +func TestSendMessage_POST_DuplicatationDisabledOnFifoQueue(t *testing.T) { + done := make(chan bool) + go PeriodicTasks(1*time.Second, done) + + // create a queue + req, err := http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "no-dup-testing.fifo") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ := CreateQueueV1(req) + + assert.Equal(t, status, http.StatusOK) + + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form = url.Values{} + form.Add("Action", "SendMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/no-dup-testing.fifo") + form.Add("MessageBody", "Test1") + form.Add("MessageDeduplicationId", "123") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = SendMessageV1(req) + + // Check the status code is what we expect. + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", + status, http.StatusOK) + } + if len(models.SyncQueues.Queues["no-dup-testing.fifo"].Messages) == 0 { + t.Fatal("there should be 1 message in queue") + } + + form = url.Values{} + form.Add("Action", "SendMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/no-dup-testing.fifo") + form.Add("MessageBody", "Test2") + form.Add("MessageDeduplicationId", "123") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = SendMessageV1(req) + + // Check the status code is what we expect. + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", + status, http.StatusOK) + } + if len(models.SyncQueues.Queues["no-dup-testing.fifo"].Messages) != 2 { + t.Fatal("there should be 2 message in queue") + } + done <- true +} + +func TestSendMessage_POST_DuplicatationEnabledOnFifoQueue(t *testing.T) { + done := make(chan bool) + go PeriodicTasks(1*time.Second, done) + + // create a queue + req, err := http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "dup-testing.fifo") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ := CreateQueueV1(req) + + assert.Equal(t, status, http.StatusOK) + + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + models.SyncQueues.Queues["dup-testing.fifo"].EnableDuplicates = true + + form = url.Values{} + form.Add("Action", "SendMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/dup-testing.fifo") + form.Add("MessageBody", "Test1") + form.Add("MessageDeduplicationId", "123") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = SendMessageV1(req) + + // Check the status code is what we expect. + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", + status, http.StatusOK) + } + if len(models.SyncQueues.Queues["dup-testing.fifo"].Messages) == 0 { + t.Fatal("there should be 1 message in queue") + } + + form = url.Values{} + form.Add("Action", "SendMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/dup-testing.fifo") + form.Add("MessageBody", "Test2") + form.Add("MessageDeduplicationId", "123") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = SendMessageV1(req) + + // Check the status code is what we expect. + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", + status, http.StatusOK) + } + if len(models.SyncQueues.Queues["dup-testing.fifo"].Messages) != 1 { + t.Fatal("there should be 1 message in queue") + } + if body := models.SyncQueues.Queues["dup-testing.fifo"].Messages[0].MessageBody; string(body) == "Test2" { + t.Fatal("duplicate message should not be added to queue") + } + done <- true +} + +func TestSendMessage_POST_DelaySeconds(t *testing.T) { + // create a queue + req, err := http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "sendmessage-delay") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ := CreateQueueV1(req) + + assert.Equal(t, status, http.StatusOK) + + // send a message + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + form = url.Values{} + form.Add("Action", "SendMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/sendmessage-delay") + form.Add("MessageBody", "1") + form.Add("DelaySeconds", "2") + form.Add("Version", "2012-11-05") + req.PostForm = form + + status, _ = SendMessageV1(req) + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", + status, http.StatusOK) + } + + // receive message before delay is up + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + form = url.Values{} + form.Add("Action", "ReceiveMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/sendmessage-delay") + form.Add("Version", "2012-11-05") + req.PostForm = form + status, _ = ReceiveMessageV1(req) + assert.Equal(t, status, http.StatusOK) + + // receive message with wait should return after delay + req, err = http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + form = url.Values{} + form.Add("Action", "ReceiveMessage") + form.Add("QueueUrl", "http://localhost:4100/queue/sendmessage-delay") + form.Add("WaitTimeSeconds", "10") + form.Add("Version", "2012-11-05") + req.PostForm = form + start := time.Now() + status, _ = ReceiveMessageV1(req) + elapsed := time.Since(start) + assert.Equal(t, status, http.StatusOK) + if elapsed < 1*time.Second { + t.Errorf("handler didn't wait at all") + } + if elapsed > 4*time.Second { + t.Errorf("handler didn't need to wait all WaitTimeSeconds=10, only DelaySeconds=2") + } +} + +func TestCreateErrorResponseV1(t *testing.T) { + expectedResponse := models.ErrorResponse{ + Result: models.ErrorResult{ + Type: "Not Found", + Code: "AWS.SimpleQueueService.NonExistentQueue", + Message: "The specified queue does not exist for this wsdl version.", + }, + RequestId: "00000000-0000-0000-0000-000000000000", + } + status, response := utils.CreateErrorResponseV1("QueueNotFound", true) + + assert.Equal(t, http.StatusBadRequest, status) + assert.Equal(t, expectedResponse, response) +} + +// waitTimeout waits for the waitgroup for the specified max timeout. +// Returns true if waiting timed out. +// credits: https://stackoverflow.com/questions/32840687/timeout-for-waitgroup-wait +func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { + c := make(chan struct{}) + go func() { + defer close(c) + wg.Wait() + }() + select { + case <-c: + return false // completed normally + case <-time.After(timeout): + return true // timed out + } +} diff --git a/shared-sqs/app/gosqs/list_queues.go b/shared-sqs/app/gosqs/list_queues.go new file mode 100644 index 0000000..ecf613f --- /dev/null +++ b/shared-sqs/app/gosqs/list_queues.go @@ -0,0 +1,45 @@ +package gosqs + +import ( + "net/http" + "strings" + + "shared-sqs/app/utils" + + "shared-sqs/app/models" + + "shared-sqs/app/interfaces" + log "github.com/sirupsen/logrus" +) + +// TODO - set up MaxResults, NextToken request params +// +// https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ListQueues.html +func ListQueuesV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewListQueuesRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, true) + if !ok { + log.Error("Invalid Request - ListQueuesV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + log.Info("Listing Queues") + queueUrls := make([]string, 0) + models.SyncQueues.Lock() + for _, queue := range models.SyncQueues.Queues { + if strings.HasPrefix(queue.Name, requestBody.QueueNamePrefix) { + queueUrls = append(queueUrls, queue.URL) + } + } + models.SyncQueues.Unlock() + + respStruct := models.ListQueuesResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + Result: models.ListQueuesResult{ + QueueUrls: queueUrls, + }, + } + + return http.StatusOK, respStruct +} diff --git a/shared-sqs/app/gosqs/list_queues_test.go b/shared-sqs/app/gosqs/list_queues_test.go new file mode 100644 index 0000000..ab85647 --- /dev/null +++ b/shared-sqs/app/gosqs/list_queues_test.go @@ -0,0 +1,116 @@ +package gosqs + +import ( + "fmt" + "net/http" + "testing" + + "shared-sqs/app/test" + + "shared-sqs/app/conf" + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/stretchr/testify/assert" +) + +func TestListQueuesV1_success(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.ListQueueRequest) + *v = models.ListQueueRequest{} + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := ListQueuesV1(r) + r1 := response.(models.ListQueuesResponse) + + assert.Equal(t, http.StatusOK, code) + assert.Contains(t, r1.Result.QueueUrls, fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1")) + assert.Contains(t, r1.Result.QueueUrls, fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue2")) + assert.Contains(t, r1.Result.QueueUrls, fmt.Sprintf("%s/%s", fixtures.BASE_URL, "dead-letter-queue1")) +} + +func TestListQueuesV1_success_no_queues(t *testing.T) { + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.ListQueueRequest) + *v = models.ListQueueRequest{} + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := ListQueuesV1(r) + r1 := response.(models.ListQueuesResponse) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, r1.Result.QueueUrls, []string{}) +} + +func TestListQueuesV1_success_with_queue_name_prefix(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.ListQueueRequest) + *v = models.ListQueueRequest{QueueNamePrefix: "dead-letter"} + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := ListQueuesV1(r) + r1 := response.(models.ListQueuesResponse) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, []string{fmt.Sprintf("%s/%s", fixtures.BASE_URL, "dead-letter-queue1")}, r1.Result.QueueUrls) +} + +func TestListQueuesV1_success_with_queue_name_prefix_no_matching_queues(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.ListQueueRequest) + *v = models.ListQueueRequest{QueueNamePrefix: "garbage"} + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := ListQueuesV1(r) + r1 := response.(models.ListQueuesResponse) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, []string{}, r1.Result.QueueUrls) +} + +func TestListQueuesV1_request_transformer_error(t *testing.T) { + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := ListQueuesV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} diff --git a/shared-sqs/app/gosqs/purge_queue.go b/shared-sqs/app/gosqs/purge_queue.go new file mode 100644 index 0000000..31b80a9 --- /dev/null +++ b/shared-sqs/app/gosqs/purge_queue.go @@ -0,0 +1,42 @@ +package gosqs + +import ( + "net/http" + "strings" + "time" + + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + + log "github.com/sirupsen/logrus" +) + +func PurgeQueueV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewPurgeQueueRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - PurgeQueueV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + uriSegments := strings.Split(requestBody.QueueUrl, "/") + queueName := uriSegments[len(uriSegments)-1] + + models.SyncQueues.Lock() + defer models.SyncQueues.Unlock() + if _, ok := models.SyncQueues.Queues[queueName]; !ok { + log.Errorf("Purge Queue: %s, queue does not exist!!!", queueName) + return utils.CreateErrorResponseV1("QueueNotFound", true) + } + + log.Infof("Purging Queue: %s", queueName) + models.SyncQueues.Queues[queueName].Messages = nil + models.SyncQueues.Queues[queueName].Duplicates = make(map[string]time.Time) + + respStruct := models.PurgeQueueResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + return http.StatusOK, respStruct +} diff --git a/shared-sqs/app/gosqs/purge_queue_test.go b/shared-sqs/app/gosqs/purge_queue_test.go new file mode 100644 index 0000000..9677dab --- /dev/null +++ b/shared-sqs/app/gosqs/purge_queue_test.go @@ -0,0 +1,126 @@ +package gosqs + +import ( + "fmt" + "net/http" + "testing" + "time" + + "shared-sqs/app/test" + + "shared-sqs/app/conf" + + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/stretchr/testify/assert" +) + +func TestPurgeQueueV1_success(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.PurgeQueueRequest) + *v = models.PurgeQueueRequest{ + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + return true + } + + // Put a message on the queue + targetQueue := models.SyncQueues.Queues["unit-queue1"] + models.SyncQueues.Lock() + targetQueue.Messages = []models.SqsMessage{models.SqsMessage{}} + targetQueue.Duplicates = map[string]time.Time{ + "dedupe-id": time.Now(), + } + models.SyncQueues.Unlock() + + expectedResponse := models.PurgeQueueResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := PurgeQueueV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, expectedResponse, response) + + assert.Nil(t, targetQueue.Messages) + assert.Equal(t, map[string]time.Time{}, targetQueue.Duplicates) +} + +func TestPurgeQueueV1_success_no_messages_on_queue(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.PurgeQueueRequest) + *v = models.PurgeQueueRequest{ + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + return true + } + + expectedResponse := models.PurgeQueueResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := PurgeQueueV1(r) + + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, expectedResponse, response) + + targetQueue := models.SyncQueues.Queues["unit-queue1"] + assert.Nil(t, targetQueue.Messages) + assert.Equal(t, map[string]time.Time{}, targetQueue.Duplicates) +} + +func TestPurgeQueueV1_request_transformer_error(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := PurgeQueueV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} + +func TestPurgeQueueV1_requested_queue_does_not_exist(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.PurgeQueueRequest) + *v = models.PurgeQueueRequest{ + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "garbage"), + } + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := PurgeQueueV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} diff --git a/shared-sqs/app/gosqs/queue_attributes.go b/shared-sqs/app/gosqs/queue_attributes.go new file mode 100644 index 0000000..aa1f352 --- /dev/null +++ b/shared-sqs/app/gosqs/queue_attributes.go @@ -0,0 +1,47 @@ +package gosqs + +import ( + "fmt" + "strings" + + log "github.com/sirupsen/logrus" + + "shared-sqs/app/models" +) + +// TODO - Support: +// - attr.MessageRetentionPeriod +// - attr.Policy +// - attr.RedriveAllowPolicy +func setQueueAttributesV1(q *models.Queue, attr models.QueueAttributes) error { + // FIXME - are there better places to put these bottom-limit validations? + if attr.DelaySeconds >= 0 { + q.DelaySeconds = attr.DelaySeconds.Int() + } + if attr.MaximumMessageSize >= 0 { + q.MaximumMessageSize = attr.MaximumMessageSize.Int() + } + // TODO - bottom limit should be the AWS limits + // The following 2 don't support zero values + if attr.MessageRetentionPeriod > 0 { + q.MessageRetentionPeriod = attr.MessageRetentionPeriod.Int() + } + if attr.ReceiveMessageWaitTimeSeconds > 0 { + q.ReceiveMessageWaitTimeSeconds = attr.ReceiveMessageWaitTimeSeconds.Int() + } + if attr.VisibilityTimeout >= 0 { + q.VisibilityTimeout = attr.VisibilityTimeout.Int() + } + if attr.RedrivePolicy != (models.RedrivePolicy{}) { + arnArray := strings.Split(attr.RedrivePolicy.DeadLetterTargetArn, ":") + queueName := arnArray[len(arnArray)-1] + deadLetterQueue, ok := models.SyncQueues.Queues[queueName] + if !ok { + log.Error("Invalid RedrivePolicy Attribute") + return fmt.Errorf("InvalidAttributeValue") + } + q.DeadLetterQueue = deadLetterQueue + q.MaxReceiveCount = attr.RedrivePolicy.MaxReceiveCount.Int() + } + return nil +} diff --git a/shared-sqs/app/gosqs/queue_attributes_test.go b/shared-sqs/app/gosqs/queue_attributes_test.go new file mode 100644 index 0000000..d79058e --- /dev/null +++ b/shared-sqs/app/gosqs/queue_attributes_test.go @@ -0,0 +1,123 @@ +package gosqs + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "shared-sqs/app/models" +) + +func TestSetQueueAttributesV1_success_no_redrive_policy(t *testing.T) { + var emptyQueue *models.Queue + q := &models.Queue{} + attrs := models.QueueAttributes{ + DelaySeconds: 1, + MaximumMessageSize: 2, + MessageRetentionPeriod: 3, + ReceiveMessageWaitTimeSeconds: 4, + VisibilityTimeout: 5, + } + err := setQueueAttributesV1(q, attrs) + + assert.Nil(t, err) + assert.Equal(t, 1, q.DelaySeconds) + assert.Equal(t, 2, q.MaximumMessageSize) + assert.Equal(t, 3, q.MessageRetentionPeriod) + assert.Equal(t, 4, q.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, 5, q.VisibilityTimeout) + assert.Equal(t, emptyQueue, q.DeadLetterQueue) + assert.Equal(t, 0, q.MaxReceiveCount) +} + +func TestSetQueueAttributesV1_success_no_request_attributes(t *testing.T) { + var emptyQueue *models.Queue + q := &models.Queue{} + attrs := models.QueueAttributes{} + err := setQueueAttributesV1(q, attrs) + + assert.Nil(t, err) + assert.Equal(t, 0, q.DelaySeconds) + assert.Equal(t, 0, q.MaximumMessageSize) + assert.Equal(t, 0, q.MessageRetentionPeriod) + assert.Equal(t, 0, q.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, 0, q.VisibilityTimeout) + assert.Equal(t, emptyQueue, q.DeadLetterQueue) + assert.Equal(t, 0, q.MaxReceiveCount) +} + +func TestSetQueueAttributesV1_success_can_set_0_values_where_applicable(t *testing.T) { + var emptyQueue *models.Queue + q := &models.Queue{ + DelaySeconds: 1, + MaximumMessageSize: 2, + MessageRetentionPeriod: 3, + ReceiveMessageWaitTimeSeconds: 4, + VisibilityTimeout: 5, + } + attrs := models.QueueAttributes{} + err := setQueueAttributesV1(q, attrs) + + assert.Nil(t, err) + assert.Equal(t, 0, q.DelaySeconds) + assert.Equal(t, 0, q.MaximumMessageSize) + assert.Equal(t, 3, q.MessageRetentionPeriod) + assert.Equal(t, 4, q.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, 0, q.VisibilityTimeout) + assert.Equal(t, emptyQueue, q.DeadLetterQueue) + assert.Equal(t, 0, q.MaxReceiveCount) +} + +func TestSetQueueAttributesV1_success_with_redrive_policy(t *testing.T) { + defer func() { + models.ResetApp() + }() + + existingQueueName := "existing-queue" + existingQueue := &models.Queue{Name: existingQueueName} + models.SyncQueues.Queues[existingQueueName] = existingQueue + + q := &models.Queue{} + attrs := models.QueueAttributes{ + DelaySeconds: 1, + MaximumMessageSize: 2, + MessageRetentionPeriod: 3, + ReceiveMessageWaitTimeSeconds: 4, + VisibilityTimeout: 5, + RedrivePolicy: models.RedrivePolicy{ + MaxReceiveCount: 10, + DeadLetterTargetArn: fmt.Sprintf("arn:aws:sqs:region:account-id:%s", existingQueueName), + }, + } + err := setQueueAttributesV1(q, attrs) + + assert.Nil(t, err) + assert.Equal(t, 1, q.DelaySeconds) + assert.Equal(t, 2, q.MaximumMessageSize) + assert.Equal(t, 3, q.MessageRetentionPeriod) + assert.Equal(t, 4, q.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, 5, q.VisibilityTimeout) + assert.Equal(t, existingQueue, q.DeadLetterQueue) + assert.Equal(t, 10, q.MaxReceiveCount) +} + +func TestSetQueueAttributesV1_error_redrive_policy_targets_missing_queue(t *testing.T) { + existingQueueName := "existing-queue" + + q := &models.Queue{} + attrs := models.QueueAttributes{ + DelaySeconds: 1, + MaximumMessageSize: 2, + MessageRetentionPeriod: 3, + ReceiveMessageWaitTimeSeconds: 4, + VisibilityTimeout: 5, + RedrivePolicy: models.RedrivePolicy{ + MaxReceiveCount: 10, + DeadLetterTargetArn: fmt.Sprintf("arn:aws:sqs:region:account-id:%s", existingQueueName), + }, + } + err := setQueueAttributesV1(q, attrs) + + assert.Error(t, err) +} diff --git a/shared-sqs/app/gosqs/receive_message.go b/shared-sqs/app/gosqs/receive_message.go new file mode 100644 index 0000000..99d4bba --- /dev/null +++ b/shared-sqs/app/gosqs/receive_message.go @@ -0,0 +1,164 @@ +package gosqs + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" +) + +// TODO - Admiral-Piett - could we refactor the way we hide messages? Change data structure to a queue +// organized by "reveal time" or a map with the key being a timestamp of when it could be shown? +// Ordered Map - https://github.com/elliotchance/orderedmap +func ReceiveMessageV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewReceiveMessageRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - ReceiveMessageV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + maxNumberOfMessages := requestBody.MaxNumberOfMessages + if maxNumberOfMessages == 0 { + maxNumberOfMessages = 1 + } + + queueName := "" + if requestBody.QueueUrl == "" { + vars := mux.Vars(req) + queueName = vars["queueName"] + } else { + uriSegments := strings.Split(requestBody.QueueUrl, "/") + queueName = uriSegments[len(uriSegments)-1] + } + + if _, ok := models.SyncQueues.Queues[queueName]; !ok { + return utils.CreateErrorResponseV1("QueueNotFound", true) + } + + var messages []*models.ResultMessage + respStruct := models.ReceiveMessageResponse{} + + waitTimeSeconds := requestBody.WaitTimeSeconds + if waitTimeSeconds == 0 { + models.SyncQueues.RLock() + waitTimeSeconds = models.SyncQueues.Queues[queueName].ReceiveMessageWaitTimeSeconds + models.SyncQueues.RUnlock() + } + + loops := waitTimeSeconds * 10 + for loops > 0 { + models.SyncQueues.RLock() + _, queueFound := models.SyncQueues.Queues[queueName] + if !queueFound { + models.SyncQueues.RUnlock() + return utils.CreateErrorResponseV1("QueueNotFound", true) + } + messageFound := len(models.SyncQueues.Queues[queueName].Messages)-numberOfHiddenMessagesInQueue(*models.SyncQueues.Queues[queueName]) != 0 + models.SyncQueues.RUnlock() + if !messageFound { + continueTimer := time.NewTimer(100 * time.Millisecond) + select { + case <-req.Context().Done(): + continueTimer.Stop() + return http.StatusOK, models.ReceiveMessageResponse{ + Xmlns: models.BaseXmlns, + Result: models.ReceiveMessageResult{}, + Metadata: models.BaseResponseMetadata, + } + case <-continueTimer.C: + continueTimer.Stop() + } + loops-- + } else { + break + } + + } + log.Debugf("Getting Message from Queue:%s", queueName) + + models.SyncQueues.Lock() // Lock the Queues + defer models.SyncQueues.Unlock() // Unlock the Queues + + if len(models.SyncQueues.Queues[queueName].Messages) > 0 { + numMsg := 0 + messages = make([]*models.ResultMessage, 0) + for i := range models.SyncQueues.Queues[queueName].Messages { + if numMsg >= maxNumberOfMessages { + break + } + + if models.SyncQueues.Queues[queueName].Messages[i].ReceiptHandle != "" { + continue + } + + msg := &models.SyncQueues.Queues[queueName].Messages[i] + if !msg.IsReadyForReceipt() { + continue + } + + if models.SyncQueues.Queues[queueName].IsFIFO { + // If we got messages here it means we have not processed it yet, so get next + if models.SyncQueues.Queues[queueName].IsLocked(msg.GroupID) { + continue + } + // Otherwise lock messages for group ID + models.SyncQueues.Queues[queueName].LockGroup(msg.GroupID) + } + + randomId := uuid.NewString() + msg.ReceiptHandle = msg.Uuid + "#" + randomId + msg.ReceiptTime = time.Now().UTC() + + if requestBody.VisibilityTimeout != 0 { + msg.VisibilityTimeout = time.Now().Add(time.Duration(requestBody.VisibilityTimeout) * time.Second) + } else { + msg.VisibilityTimeout = time.Now().Add(time.Duration(models.SyncQueues.Queues[queueName].VisibilityTimeout) * time.Second) + } + + messages = append(messages, buildResultMessage(msg)) + + numMsg++ + } + + respStruct = models.ReceiveMessageResponse{ + "http://queue.amazonaws.com/doc/2012-11-05/", + models.ReceiveMessageResult{ + Messages: messages, + }, + models.ResponseMetadata{ + RequestId: "00000000-0000-0000-0000-000000000000", + }, + } + } else { + log.Warning("No messages in Queue:", queueName) + respStruct = models.ReceiveMessageResponse{Xmlns: "http://queue.amazonaws.com/doc/2012-11-05/", Result: models.ReceiveMessageResult{}, Metadata: models.ResponseMetadata{RequestId: "00000000-0000-0000-0000-000000000000"}} + } + + return http.StatusOK, respStruct +} + +func buildResultMessage(m *models.SqsMessage) *models.ResultMessage { + return &models.ResultMessage{ + MessageId: m.Uuid, + Body: m.MessageBody, + ReceiptHandle: m.ReceiptHandle, + MD5OfBody: utils.GetMD5Hash(m.MessageBody), + MD5OfMessageAttributes: m.MD5OfMessageAttributes, + MessageAttributes: m.MessageAttributes, + Attributes: map[string]string{ + "ApproximateFirstReceiveTimestamp": fmt.Sprintf("%d", m.ReceiptTime.UnixNano()/int64(time.Millisecond)), + "SenderId": models.CurrentEnvironment.AccountID, + "ApproximateReceiveCount": fmt.Sprintf("%d", m.NumberOfReceives+1), + "SentTimestamp": fmt.Sprintf("%d", time.Now().UTC().UnixNano()/int64(time.Millisecond)), + }, + } +} diff --git a/shared-sqs/app/gosqs/receive_message_test.go b/shared-sqs/app/gosqs/receive_message_test.go new file mode 100644 index 0000000..d8dd970 --- /dev/null +++ b/shared-sqs/app/gosqs/receive_message_test.go @@ -0,0 +1,404 @@ +package gosqs + +import ( + "context" + "fmt" + "net/http" + "sync" + "testing" + "time" + + "shared-sqs/app/test" + + "shared-sqs/app/fixtures" + "shared-sqs/app/models" + "github.com/stretchr/testify/assert" +) + +// TODO Admiral-Piett - fix all these tests, we don't want to be waiting for 5 seconds plus like this. +func TestReceiveMessageV1_with_WaitTimeEnforced(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + }() + + q := &models.Queue{ + Name: "waiting-queue", + ReceiveMessageWaitTimeSeconds: 2, + //MaximumMessageSize: 262144, + } + models.SyncQueues.Queues["waiting-queue"] = q + + // receive message ensure delay + _, r := test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: "http://localhost:4100/queue/waiting-queue", + }, true) + + start := time.Now() + status, response := ReceiveMessageV1(r) + elapsed := time.Since(start) + + assert.Equal(t, http.StatusOK, status) + if elapsed < 2*time.Second { + t.Fatalf("handler didn't wait ReceiveMessageWaitTimeSeconds %s", response) + } + + // mock sending a message + q.Messages = append(q.Messages, models.SqsMessage{MessageBody: "1"}) + + // receive message + _, r = test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: "http://localhost:4100/queue/waiting-queue", + }, true) + start = time.Now() + status, resp := ReceiveMessageV1(r) + elapsed = time.Since(start) + + assert.Equal(t, http.StatusOK, status) + if elapsed > 1*time.Second { + t.Fatal("handler waited when message was available, expected not to wait") + } + + assert.Equal(t, "1", string(resp.GetResult().(models.ReceiveMessageResult).Messages[0].Body)) +} + +func TestReceiveMessageV1_CanceledByClient(t *testing.T) { + // create a queue + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + }() + + q := &models.Queue{ + Name: "cancel-queue", + ReceiveMessageWaitTimeSeconds: 20, + } + models.SyncQueues.Queues["cancel-queue"] = q + + var wg sync.WaitGroup + ctx, cancelReceive := context.WithCancel(context.Background()) + + wg.Add(1) + go func() { + defer wg.Done() + // receive message (that will be canceled) + _, r := test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: "http://localhost:4100/queue/cancel-queue", + }, true) + r = r.WithContext(ctx) + + status, resp := ReceiveMessageV1(r) + assert.Equal(t, http.StatusOK, status) + + if len(resp.GetResult().(models.ReceiveMessageResult).Messages) != 0 { + t.Fatal("expecting this ReceiveMessage() to not pickup this message as it should canceled before the Send()") + } + }() + time.Sleep(100 * time.Millisecond) // let enought time for the Receive go to wait mode + cancelReceive() // cancel the first ReceiveMessage(), make sure it will not pickup the sent message below + time.Sleep(5 * time.Millisecond) + + // send a message + _, r := test.GenerateRequestInfo("POST", "/", models.SendMessageRequest{ + QueueUrl: "http://localhost:4100/queue/cancel-queue", + MessageBody: "12345", + }, true) + status, _ := SendMessageV1(r) + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", status, http.StatusOK) + } + + // receive message + _, r = test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: "http://localhost:4100/queue/cancel-queue", + }, true) + start := time.Now() + status, resp := ReceiveMessageV1(r) + assert.Equal(t, http.StatusOK, status) + elapsed := time.Since(start) + + result, ok := resp.GetResult().(models.ReceiveMessageResult) + if !ok { + t.Fatal("handler should return a message") + } + + if len(result.Messages) == 0 || string(result.Messages[0].Body) == "12345\n" { + t.Fatal("handler should return a message") + } + if elapsed > 1*time.Second { + t.Fatal("handler waited when message was available, expected not to wait") + } + + if timedout := waitTimeout(&wg, 2*time.Second); timedout { + t.Errorf("expected ReceiveMessage() in goroutine to exit quickly due to cancelReceive() called") + } +} + +func TestReceiveMessageV1_with_DelaySeconds(t *testing.T) { + // create a queue + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + }() + + q := &models.Queue{ + Name: "delay-seconds-queue", + DelaySeconds: 2, + } + models.SyncQueues.Queues["delay-seconds-queue"] = q + + // send a message + _, r := test.GenerateRequestInfo("POST", "/", models.SendMessageRequest{ + QueueUrl: "http://localhost:4100/queue/delay-seconds-queue", + MessageBody: "1", + }, true) + status, _ := SendMessageV1(r) + if status != http.StatusOK { + t.Errorf("handler returned wrong status code: got \n%v want %v", status, http.StatusOK) + } + + // receive message before delay is up + _, r = test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{QueueUrl: "http://localhost:4100/queue/delay-seconds-queue"}, true) + status, _ = ReceiveMessageV1(r) + assert.Equal(t, http.StatusOK, status) + + // receive message with wait should return after delay + _, r = test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: "http://localhost:4100/queue/delay-seconds-queue", + WaitTimeSeconds: 10, + }, true) + start := time.Now() + status, _ = ReceiveMessageV1(r) + elapsed := time.Since(start) + assert.Equal(t, http.StatusOK, status) + if elapsed < 1*time.Second { + t.Errorf("handler didn't wait at all") + } + if elapsed > 4*time.Second { + t.Errorf("handler didn't need to wait all WaitTimeSeconds=10, only DelaySeconds=2") + } +} + +func TestReceiveMessageV1_with_MessageAttributes(t *testing.T) { + // create a queue + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + }() + + q := &models.Queue{Name: "waiting-queue"} + models.SyncQueues.Queues["waiting-queue"] = q + + // send a message + q.Messages = append(q.Messages, models.SqsMessage{ + MessageBody: "1", + MessageAttributes: map[string]models.MessageAttribute{ + "TestMessageAttrName": { + DataType: "String", + StringValue: "TestMessageAttrValue", + }, + }, + }) + + // receive message + _, r := test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{QueueUrl: "http://localhost:4100/queue/waiting-queue"}, true) + status, resp := ReceiveMessageV1(r) + result := resp.GetResult().(models.ReceiveMessageResult) + + assert.NotEmpty(t, result.Messages[0].Attributes["ApproximateFirstReceiveTimestamp"]) + assert.NotEmpty(t, result.Messages[0].Attributes["SenderId"]) + assert.NotEmpty(t, result.Messages[0].Attributes["ApproximateReceiveCount"]) + assert.NotEmpty(t, result.Messages[0].Attributes["SentTimestamp"]) + + assert.Equal(t, http.StatusOK, status) + assert.Equal(t, "1", string(result.Messages[0].Body)) + assert.Equal(t, 1, len(result.Messages[0].MessageAttributes)) + assert.Equal(t, "String", result.Messages[0].MessageAttributes["TestMessageAttrName"].DataType) + assert.Equal(t, "TestMessageAttrValue", result.Messages[0].MessageAttributes["TestMessageAttrName"].StringValue) +} + +func TestReceiveMessageV1_request_transformer_error(t *testing.T) { + // TODO +} + +func TestReceiveMessageV1_with_CustomVisibilityTimeout(t *testing.T) { + // create a queue + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + }() + + // Create a queue with a default visibility timeout of 30 seconds + q := &models.Queue{ + Name: "custom-visibility-queue", + VisibilityTimeout: 30, + } + models.SyncQueues.Queues["custom-visibility-queue"] = q + + // Add a message to the queue + q.Messages = append(q.Messages, models.SqsMessage{ + MessageBody: "test-message", + Uuid: "test-uuid", + }) + + // Test 1: Receive message with custom visibility timeout + customTimeout := 60 // 60 seconds + _, r := test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: "http://localhost:4100/queue/custom-visibility-queue", + VisibilityTimeout: customTimeout, + }, true) + + status, resp := ReceiveMessageV1(r) + assert.Equal(t, http.StatusOK, status) + + result := resp.GetResult().(models.ReceiveMessageResult) + assert.Equal(t, 1, len(result.Messages)) + assert.Equal(t, "test-message", string(result.Messages[0].Body)) + + // Verify the message in the queue has the custom visibility timeout + // We can't directly check the exact time, but we can verify it's not using the queue's default + // by checking that the visibility timeout is greater than now + default timeout - 1 second + // and less than now + custom timeout + 1 second + now := time.Now() + defaultExpiry := now.Add(time.Duration(q.VisibilityTimeout) * time.Second) + customExpiry := now.Add(time.Duration(customTimeout) * time.Second) + + // The first message should have the custom visibility timeout + msgVisibilityTimeout := q.Messages[0].VisibilityTimeout + assert.True(t, msgVisibilityTimeout.After(defaultExpiry.Add(-1*time.Second)), + "Message visibility timeout should be greater than default timeout") + assert.True(t, msgVisibilityTimeout.Before(customExpiry.Add(1*time.Second)), + "Message visibility timeout should be less than custom timeout + 1 second") + + // Test 2: Reset the queue and test with zero visibility timeout (should use queue default) + models.SyncQueues.Queues["custom-visibility-queue"] = &models.Queue{ + Name: "custom-visibility-queue", + VisibilityTimeout: 30, + } + q = models.SyncQueues.Queues["custom-visibility-queue"] + q.Messages = append(q.Messages, models.SqsMessage{ + MessageBody: "test-message-2", + Uuid: "test-uuid-2", + }) + + // Receive message with zero visibility timeout (should use queue default) + _, r = test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: "http://localhost:4100/queue/custom-visibility-queue", + VisibilityTimeout: 0, // Zero should use queue default + }, true) + + status, resp = ReceiveMessageV1(r) + assert.Equal(t, http.StatusOK, status) + + // Verify the message in the queue has the default visibility timeout + now = time.Now() + defaultExpiry = now.Add(time.Duration(q.VisibilityTimeout) * time.Second) + + // The message should have the default visibility timeout + msgVisibilityTimeout = q.Messages[0].VisibilityTimeout + assert.True(t, msgVisibilityTimeout.After(defaultExpiry.Add(-1*time.Second)), + "Message visibility timeout should be greater than default timeout - 1 second") + assert.True(t, msgVisibilityTimeout.Before(defaultExpiry.Add(1*time.Second)), + "Message visibility timeout should be less than default timeout + 1 second") +} + +func TestReceiveMessageV1_FIFOSecondMessageAvailableAfterDelete(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + }() + + queueName := "fifo-delay-queue" + queueURL := fmt.Sprintf("http://localhost:4100/queue/%s", queueName) + now := time.Now().Add(-1 * time.Minute) + + // create a queue with a visibility timeout of 10 seconds + q := &models.Queue{ + Name: queueName, + VisibilityTimeout: 10, + IsFIFO: true, + FIFOMessages: map[string]int{}, + FIFOSequenceNumbers: map[string]int{}, + Duplicates: map[string]time.Time{}, + Messages: []models.SqsMessage{ + { + MessageBody: "first", + Uuid: "first-uuid", + GroupID: "company#worker", + SentTime: now, + }, + { + MessageBody: "second", + Uuid: "second-uuid", + GroupID: "company#worker", + SentTime: now, + }, + }, + } + models.SyncQueues.Queues[queueName] = q + + // receive the first FIFO message only + _, r := test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: queueURL, + MaxNumberOfMessages: 1, + }, true) + status, resp := ReceiveMessageV1(r) + assert.Equal(t, http.StatusOK, status) + result := resp.GetResult().(models.ReceiveMessageResult) + if len(result.Messages) != 1 { + t.Fatalf("expected to receive the first FIFO message, got %d", len(result.Messages)) + } + assert.Equal(t, "first", result.Messages[0].Body) + + firstReceipt := result.Messages[0].ReceiptHandle + + // verify the second FIFO message is blocked while the first is in flight + _, r = test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: queueURL, + }, true) + status, resp = ReceiveMessageV1(r) + assert.Equal(t, http.StatusOK, status) + result = resp.GetResult().(models.ReceiveMessageResult) + if len(result.Messages) != 0 { + t.Fatalf("expected no FIFO message while the first is outstanding, got %d", len(result.Messages)) + } + + // delete the first FIFO message + _, deleteReq := test.GenerateRequestInfo("POST", "/", models.DeleteMessageRequest{ + QueueUrl: queueURL, + ReceiptHandle: firstReceipt, + }, true) + deleteStatus, _ := DeleteMessageV1(deleteReq) + assert.Equal(t, http.StatusOK, deleteStatus) + + // receive the second FIFO message and ensure it does not wait for full 10 second visibility timeout + _, r = test.GenerateRequestInfo("POST", "/", models.ReceiveMessageRequest{ + QueueUrl: queueURL, + }, true) + start := time.Now() + status, resp = ReceiveMessageV1(r) + elapsed := time.Since(start) + assert.Equal(t, http.StatusOK, status) + result = resp.GetResult().(models.ReceiveMessageResult) + if len(result.Messages) != 1 { + t.Fatalf("expected second FIFO message to be available immediately, got %d", len(result.Messages)) + } + if elapsed > time.Second { + t.Fatalf("expected second FIFO message without waiting on visibility timeout, took %s", elapsed) + } + assert.Equal(t, "second", result.Messages[0].Body) + + // delete the second FIFO message + _, deleteReq = test.GenerateRequestInfo("POST", "/", models.DeleteMessageRequest{ + QueueUrl: queueURL, + ReceiptHandle: result.Messages[0].ReceiptHandle, + }, true) + deleteStatus, _ = DeleteMessageV1(deleteReq) + assert.Equal(t, http.StatusOK, deleteStatus) + + if len(q.Messages) != 0 { + t.Fatalf("expected all FIFO messages to be deleted, remaining %d", len(q.Messages)) + } +} + +// TODO - other tests diff --git a/shared-sqs/app/gosqs/send_message.go b/shared-sqs/app/gosqs/send_message.go new file mode 100644 index 0000000..7673c1f --- /dev/null +++ b/shared-sqs/app/gosqs/send_message.go @@ -0,0 +1,100 @@ +package gosqs + +import ( + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + + "shared-sqs/app/utils" + + log "github.com/sirupsen/logrus" + + "github.com/gorilla/mux" +) + +func SendMessageV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewSendMessageRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - SendMessageV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + messageBody := requestBody.MessageBody + messageGroupID := requestBody.MessageGroupId + messageDeduplicationID := requestBody.MessageDeduplicationId + + queueUrl := getQueueFromPath(requestBody.QueueUrl, req.URL.String()) + + queueName := "" + if queueUrl == "" { + // TODO: Remove this query param logic if it's not still valid or something + vars := mux.Vars(req) + queueName = vars["queueName"] + } else { + uriSegments := strings.Split(queueUrl, "/") + queueName = uriSegments[len(uriSegments)-1] + } + + if _, ok := models.SyncQueues.Queues[queueName]; !ok { + // Queue does not exist + return utils.CreateErrorResponseV1("QueueNotFound", true) + } + + if models.SyncQueues.Queues[queueName].MaximumMessageSize > 0 && + len(messageBody) > models.SyncQueues.Queues[queueName].MaximumMessageSize { + // Message size is too big + return utils.CreateErrorResponseV1("MessageTooBig", true) + } + + delaySecs := models.SyncQueues.Queues[queueName].DelaySeconds + if requestBody.DelaySeconds != 0 { + delaySecs = requestBody.DelaySeconds + } + + log.Debugf("Putting Message in Queue: [%s]", queueName) + msg := models.SqsMessage{MessageBody: messageBody} + if len(requestBody.MessageAttributes) > 0 { + msg.MessageAttributes = requestBody.MessageAttributes + msg.MD5OfMessageAttributes = utils.HashAttributes(requestBody.MessageAttributes) + } + msg.MD5OfMessageBody = utils.GetMD5Hash(messageBody) + msg.Uuid = uuid.NewString() + msg.GroupID = messageGroupID + msg.DeduplicationID = messageDeduplicationID + msg.SentTime = time.Now() + msg.DelaySecs = delaySecs + + models.SyncQueues.Lock() + fifoSeqNumber := "" + if models.SyncQueues.Queues[queueName].IsFIFO { + fifoSeqNumber = models.SyncQueues.Queues[queueName].NextSequenceNumber(messageGroupID) + } + + if !models.SyncQueues.Queues[queueName].IsDuplicate(messageDeduplicationID) { + models.SyncQueues.Queues[queueName].Messages = append(models.SyncQueues.Queues[queueName].Messages, msg) + } else { + log.Debugf("Message with deduplicationId [%s] in queue [%s] is duplicate ", messageDeduplicationID, queueName) + } + + models.SyncQueues.Queues[queueName].InitDuplicatation(messageDeduplicationID) + models.SyncQueues.Unlock() + log.Infof("%s: Queue: %s, Message: %s\n", time.Now().Format("2006-01-02 15:04:05"), queueName, msg.MessageBody) + + respStruct := models.SendMessageResponse{ + Xmlns: models.BaseXmlns, + Result: models.SendMessageResult{ + MD5OfMessageAttributes: msg.MD5OfMessageAttributes, + MD5OfMessageBody: msg.MD5OfMessageBody, + MessageId: msg.Uuid, + SequenceNumber: fifoSeqNumber, + }, + Metadata: models.BaseResponseMetadata, + } + + return http.StatusOK, respStruct +} diff --git a/shared-sqs/app/gosqs/send_message_batch.go b/shared-sqs/app/gosqs/send_message_batch.go new file mode 100644 index 0000000..b70901a --- /dev/null +++ b/shared-sqs/app/gosqs/send_message_batch.go @@ -0,0 +1,105 @@ +package gosqs + +import ( + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" +) + +func SendMessageBatchV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewSendMessageBatchRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - SendMessageBatchV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + queueUrl := requestBody.QueueUrl + + // TODO: Remove this query param logic if it's not still valid or something + queueName := "" + if queueUrl == "" { + vars := mux.Vars(req) + queueName = vars["queueName"] + } else { + uriSegments := strings.Split(queueUrl, "/") + queueName = uriSegments[len(uriSegments)-1] + } + + if _, ok := models.SyncQueues.Queues[queueName]; !ok { + return utils.CreateErrorResponseV1("QueueNotFound", true) + } + + sendEntries := requestBody.Entries + + if len(sendEntries) == 0 { + return utils.CreateErrorResponseV1("EmptyBatchRequest", true) + } + + if len(sendEntries) > 10 { + return utils.CreateErrorResponseV1("TooManyEntriesInBatchRequest", true) + } + ids := map[string]struct{}{} + for _, v := range sendEntries { + if _, ok := ids[v.Id]; ok { + return utils.CreateErrorResponseV1("BatchEntryIdsNotDistinct", true) + } + ids[v.Id] = struct{}{} + } + + sentEntries := make([]models.SendMessageBatchResultEntry, 0) + log.Debug("Putting Message in Queue:", queueName) + for _, sendEntry := range sendEntries { + msg := models.SqsMessage{MessageBody: sendEntry.MessageBody} + if len(sendEntry.MessageAttributes) > 0 { + msg.MessageAttributes = sendEntry.MessageAttributes + msg.MD5OfMessageAttributes = utils.HashAttributes(sendEntry.MessageAttributes) + } + msg.MD5OfMessageBody = utils.GetMD5Hash(sendEntry.MessageBody) + msg.GroupID = sendEntry.MessageGroupId + msg.DeduplicationID = sendEntry.MessageDeduplicationId + msg.Uuid = uuid.NewString() + msg.SentTime = time.Now() + models.SyncQueues.Lock() + fifoSeqNumber := "" + if models.SyncQueues.Queues[queueName].IsFIFO { + fifoSeqNumber = models.SyncQueues.Queues[queueName].NextSequenceNumber(sendEntry.MessageGroupId) + } + + if !models.SyncQueues.Queues[queueName].IsDuplicate(sendEntry.MessageDeduplicationId) { + models.SyncQueues.Queues[queueName].Messages = append(models.SyncQueues.Queues[queueName].Messages, msg) + } else { + log.Debugf("Message with deduplicationId [%s] in queue [%s] is duplicate ", sendEntry.MessageDeduplicationId, queueName) + } + + models.SyncQueues.Queues[queueName].InitDuplicatation(sendEntry.MessageDeduplicationId) + + models.SyncQueues.Unlock() + se := models.SendMessageBatchResultEntry{ + Id: sendEntry.Id, + MessageId: msg.Uuid, + MD5OfMessageBody: msg.MD5OfMessageBody, + MD5OfMessageAttributes: msg.MD5OfMessageAttributes, + SequenceNumber: fifoSeqNumber, + } + sentEntries = append(sentEntries, se) + log.Infof("%s: Queue: %s, Message: %s\n", time.Now().Format("2006-01-02 15:04:05"), queueName, msg.MessageBody) + } + + respStruct := models.SendMessageBatchResponse{ + Xmlns: models.BaseXmlns, + Result: models.SendMessageBatchResult{Entry: sentEntries}, + Metadata: models.BaseResponseMetadata, + } + + return http.StatusOK, respStruct + +} diff --git a/shared-sqs/app/gosqs/send_message_batch_test.go b/shared-sqs/app/gosqs/send_message_batch_test.go new file mode 100644 index 0000000..45abe31 --- /dev/null +++ b/shared-sqs/app/gosqs/send_message_batch_test.go @@ -0,0 +1,346 @@ +package gosqs + +import ( + "fmt" + "net/http" + "testing" + + "shared-sqs/app/conf" + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/test" + "shared-sqs/app/utils" + "github.com/stretchr/testify/assert" +) + +func TestSendMessageBatchV1_Success(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_success := models.SendMessageBatchRequest{ + Entries: []models.SendMessageBatchRequestEntry{ + { + Id: "test-msg-with-non-attribute", + MessageBody: "test%20message%20body%201", + }, + { + Id: "test-msg-with-single-attirbute", + MessageBody: "test%20message%20body%202", + MessageAttributes: map[string]models.MessageAttribute{ + "my-attribute-name": { + DataType: "String", + StringValue: "my-attribute-string-value", + }, + }, + }, + { + Id: "test-msg-with-multi-attirbute", + MessageBody: "test%20message%20body%203", + MessageAttributes: map[string]models.MessageAttribute{ + "my-attribute-name-1": { + BinaryValue: "binary-value-1", + DataType: "Binary", + }, + "my-attribute-name-2": { + DataType: "String", + StringValue: "my-attribute-string-value-2", + }, + }, + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageBatchRequest) + *v = sendMessageRequest_success + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageBatchV1(r) + sendMessageBatchResponse, ok := response.(models.SendMessageBatchResponse) + + assert.Equal(t, http.StatusOK, status) + assert.True(t, ok) + + resultEntry := sendMessageBatchResponse.Result.Entry + assert.Equal(t, 3, len(resultEntry)) + assert.Contains(t, resultEntry[0].Id, "test-msg-with-non-attribute") + assert.Contains(t, resultEntry[1].Id, "test-msg-with-single-attirbute") + assert.Contains(t, resultEntry[2].Id, "test-msg-with-multi-attirbute") + assert.Empty(t, resultEntry[0].SequenceNumber) + assert.Empty(t, resultEntry[1].SequenceNumber) + assert.Empty(t, resultEntry[2].SequenceNumber) + +} + +func TestSendMessageBatchV1_Success_Fifo_Queue(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_success := models.SendMessageBatchRequest{ + Entries: []models.SendMessageBatchRequestEntry{ + { + Id: "test_msg_001", + MessageBody: "test%20message%20body%201", + }, + { + Id: "test_msg_002", + MessageBody: "test%20message%20body%202", + }, + { + Id: "test_msg_003", + MessageBody: "test%20message%20body%203", + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "fifo-queue-1"), + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageBatchRequest) + *v = sendMessageRequest_success + return true + } + + q := &models.Queue{ + Name: "fifo-queue-1", + MaximumMessageSize: 1024, + IsFIFO: true, + } + models.SyncQueues.Queues["fifo-queue-1"] = q + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageBatchV1(r) + sendMessageBatchResponse, ok := response.(models.SendMessageBatchResponse) + + assert.Equal(t, http.StatusOK, status) + assert.True(t, ok) + + resultEntry := sendMessageBatchResponse.Result.Entry + assert.Equal(t, 3, len(resultEntry)) + assert.Contains(t, resultEntry[0].Id, "test_msg_001") + assert.NotEmpty(t, resultEntry[0].SequenceNumber) + assert.Contains(t, resultEntry[1].Id, "test_msg_002") + assert.NotEmpty(t, resultEntry[1].SequenceNumber) + assert.Contains(t, resultEntry[2].Id, "test_msg_003") + assert.NotEmpty(t, resultEntry[2].SequenceNumber) +} + +func TestSendMessageBatchV1_Error_QueueNotFound(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_not_found_queue := models.SendMessageBatchRequest{ + Entries: []models.SendMessageBatchRequestEntry{ + { + Id: "test_msg_001", + MessageBody: "test%20message%20body%201", + }, + { + Id: "test_msg_002", + MessageBody: "test%20message%20body%202", + }, + { + Id: "test_msg_003", + MessageBody: "test%20message%20body%203", + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "not-exist-queue1"), + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageBatchRequest) + *v = sendMessageRequest_not_found_queue + return true + } + + expected := models.ErrorResult{ + Type: "Not Found", + Code: "AWS.SimpleQueueService.NonExistentQueue", + Message: "The specified queue does not exist for this wsdl version.", + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageBatchV1(r) + errorResult := response.GetResult().(models.ErrorResult) + + assert.Equal(t, http.StatusBadRequest, status) + assert.Equal(t, expected, errorResult) +} + +func TestSendMessageBatchV1_Error_NoEntry(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_no_entry := models.SendMessageBatchRequest{ + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageBatchRequest) + *v = sendMessageRequest_no_entry + return true + } + + expected := models.ErrorResult{ + Type: "EmptyBatchRequest", + Code: "AWS.SimpleQueueService.EmptyBatchRequest", + Message: "The batch request doesn't contain any entries.", + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageBatchV1(r) + errorResult := response.GetResult().(models.ErrorResult) + + assert.Equal(t, http.StatusBadRequest, status) + assert.Equal(t, expected, errorResult) +} + +func TestSendMessageBatchV1_Error_IdNotDistinct(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_no_entry := models.SendMessageBatchRequest{ + Entries: []models.SendMessageBatchRequestEntry{ + { + Id: "test_msg_001", + MessageBody: "test%20message%20body%201", + }, + { + Id: "test_msg_001", + MessageBody: "test%20message%20body%202", + }, + { + Id: "test_msg_001", + MessageBody: "test%20message%20body%203", + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageBatchRequest) + *v = sendMessageRequest_no_entry + return true + } + + expected := models.ErrorResult{ + Type: "BatchEntryIdsNotDistinct", + Code: "AWS.SimpleQueueService.BatchEntryIdsNotDistinct", + Message: "Two or more batch entries in the request have the same Id.", + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageBatchV1(r) + errorResult := response.GetResult().(models.ErrorResult) + + assert.Equal(t, http.StatusBadRequest, status) + assert.Equal(t, expected, errorResult) +} + +func TestSendMessageBatchV1_Error_TooManyEntries(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_success := models.SendMessageBatchRequest{ + Entries: []models.SendMessageBatchRequestEntry{ + { + Id: "test_msg_001", + MessageBody: "test%20message%20body%201", + }, + { + Id: "test_msg_002", + MessageBody: "test%20message%20body%202", + }, + { + Id: "test_msg_003", + MessageBody: "test%20message%20body%203", + }, + { + Id: "test_msg_004", + MessageBody: "test%20message%20body%204", + }, + { + Id: "test_msg_005", + MessageBody: "test%20message%20body%205", + }, + { + Id: "test_msg_006", + MessageBody: "test%20message%20body%206", + }, + { + Id: "test_msg_007", + MessageBody: "test%20message%20body%207", + }, + { + Id: "test_msg_008", + MessageBody: "test%20message%20body%208", + }, + { + Id: "test_msg_009", + MessageBody: "test%20message%20body%209", + }, + { + Id: "test_msg_010", + MessageBody: "test%20message%20body%210", + }, + { + Id: "test_msg_011", + MessageBody: "test%20message%20body%211", + }, + }, + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageBatchRequest) + *v = sendMessageRequest_success + return true + } + + expected := models.ErrorResult{ + Type: "TooManyEntriesInBatchRequest", + Code: "AWS.SimpleQueueService.TooManyEntriesInBatchRequest", + Message: "Maximum number of entries per request are 10.", + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageBatchV1(r) + errorResult := response.GetResult().(models.ErrorResult) + + assert.Equal(t, http.StatusBadRequest, status) + assert.Equal(t, expected, errorResult) + +} + +func TestSendMessageBatchV1_Error_transformer(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := SendMessageBatchV1(r) + + assert.Equal(t, http.StatusBadRequest, code) + +} diff --git a/shared-sqs/app/gosqs/send_message_test.go b/shared-sqs/app/gosqs/send_message_test.go new file mode 100644 index 0000000..f0d70b2 --- /dev/null +++ b/shared-sqs/app/gosqs/send_message_test.go @@ -0,0 +1,220 @@ +package gosqs + +import ( + "net/http" + "testing" + "time" + + "shared-sqs/app/test" + + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/stretchr/testify/assert" +) + +func TestSendMessageV1_Success(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_success := models.SendMessageRequest{ + QueueUrl: "http://localhost:4200/new-queue-1", + MessageBody: "Test Message", + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageRequest) + *v = sendMessageRequest_success + return true + } + + q := &models.Queue{ + Name: "new-queue-1", + MaximumMessageSize: 1024, + } + models.SyncQueues.Queues["new-queue-1"] = q + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageV1(r) + + // Check the queue + assert.Equal(t, 1, len(q.Messages)) + msg := q.Messages[0] + assert.Equal(t, "Test Message", string(msg.MessageBody)) + + // Check the response + assert.Equal(t, http.StatusOK, status) + sendMessageResponse, ok := response.(models.SendMessageResponse) + assert.True(t, ok) + assert.NotEmpty(t, sendMessageResponse.Result.MD5OfMessageBody) + // No FIFO Sequence + assert.Empty(t, sendMessageResponse.Result.SequenceNumber) +} + +func TestSendMessageV1_Success_FIFOQueue(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_success := models.SendMessageRequest{ + QueueUrl: "http://localhost:4200/new-queue-1", + MessageBody: "Test Message", + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageRequest) + *v = sendMessageRequest_success + return true + } + + q := &models.Queue{ + Name: "new-queue-1", + MaximumMessageSize: 1024, + IsFIFO: true, + } + models.SyncQueues.Queues["new-queue-1"] = q + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageV1(r) + + // Check the queue + assert.Equal(t, 1, len(q.Messages)) + msg := q.Messages[0] + assert.Equal(t, "Test Message", string(msg.MessageBody)) + + // Check the response + assert.Equal(t, http.StatusOK, status) + sendMessageResponse, ok := response.(models.SendMessageResponse) + assert.True(t, ok) + assert.NotEmpty(t, sendMessageResponse.Result.MD5OfMessageBody) + // Should have FIFO Sequence + assert.NotEmpty(t, sendMessageResponse.Result.SequenceNumber) +} + +func TestSendMessageV1_Success_Deduplication(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_success := models.SendMessageRequest{ + QueueUrl: "http://localhost:4200/new-queue-1", + MessageBody: "Test Message", + MessageDeduplicationId: "1", + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageRequest) + *v = sendMessageRequest_success + return true + } + + q := &models.Queue{ + Name: "new-queue-1", + MaximumMessageSize: 1024, + IsFIFO: true, + EnableDuplicates: true, + Duplicates: make(map[string]time.Time), + } + models.SyncQueues.Queues["new-queue-1"] = q + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, _ := SendMessageV1(r) + + // Check the queue + assert.Equal(t, 1, len(q.Messages)) + // Check the response + assert.Equal(t, http.StatusOK, status) + + // Send the same message (have DeduplicationId) + status, _ = SendMessageV1(r) + // Response is "success" + assert.Equal(t, http.StatusOK, status) + // Only 1 message should be in the queue + assert.Equal(t, 1, len(q.Messages)) +} + +func TestSendMessageV1_request_transformer_error(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := SendMessageV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} + +func TestSendMessageV1_MaximumMessageSize_MessageTooBig(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_success := models.SendMessageRequest{ + QueueUrl: "http://localhost:4200/new-queue-1", + MessageBody: "Test Message", + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageRequest) + *v = sendMessageRequest_success + return true + } + + q := &models.Queue{ + Name: "new-queue-1", + MaximumMessageSize: 1, + } + models.SyncQueues.Queues["new-queue-1"] = q + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageV1(r) + + // Check the response + assert.Equal(t, http.StatusBadRequest, status) + errorResponse, ok := response.(models.ErrorResponse) + assert.True(t, ok) + assert.Equal(t, "MessageTooBig", errorResponse.Result.Type) +} + +func TestSendMessageV1_POST_QueueNonExistant(t *testing.T) { + models.CurrentEnvironment = fixtures.LOCAL_ENVIRONMENT + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + sendMessageRequest_success := models.SendMessageRequest{ + QueueUrl: "http://localhost:4200/new-queue-1", + MessageBody: "Test Message", + } + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SendMessageRequest) + *v = sendMessageRequest_success + return true + } + + // No test queue is added to app.SyncQueues + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + status, response := SendMessageV1(r) + + // Check the status code is what we expect. + assert.Equal(t, http.StatusBadRequest, status) + + // Check the response body is what we expect. + errorResponse, ok := response.(models.ErrorResponse) + assert.True(t, ok) + assert.Equal(t, "Not Found", errorResponse.Result.Type) +} diff --git a/shared-sqs/app/gosqs/set_queue_attributes.go b/shared-sqs/app/gosqs/set_queue_attributes.go new file mode 100644 index 0000000..c9de081 --- /dev/null +++ b/shared-sqs/app/gosqs/set_queue_attributes.go @@ -0,0 +1,48 @@ +package gosqs + +import ( + "net/http" + "strings" + + "shared-sqs/app/models" + "shared-sqs/app/utils" + + "shared-sqs/app/interfaces" + log "github.com/sirupsen/logrus" +) + +func SetQueueAttributesV1(req *http.Request) (int, interfaces.AbstractResponseBody) { + requestBody := models.NewSetQueueAttributesRequest() + ok := utils.REQUEST_TRANSFORMER(requestBody, req, false) + if !ok { + log.Error("Invalid Request - GetQueueAttributesV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + if requestBody.QueueUrl == "" { + log.Error("Missing QueueUrl - GetQueueAttributesV1") + return utils.CreateErrorResponseV1("InvalidParameterValue", true) + } + + // NOTE: I tore out the handling for devining the url from a param. I can't find documentation that + // that is valid any longer. + uriSegments := strings.Split(requestBody.QueueUrl, "/") + queueName := uriSegments[len(uriSegments)-1] + + log.Infof("Set Queue QueueAttributes: %s", queueName) + models.SyncQueues.Lock() + defer models.SyncQueues.Unlock() + queue, ok := models.SyncQueues.Queues[queueName] + if !ok { + log.Warningf("Get Queue URL: %s, queue does not exist!!!", queueName) + return utils.CreateErrorResponseV1("QueueNotFound", true) + } + if err := setQueueAttributesV1(queue, requestBody.Attributes); err != nil { + return utils.CreateErrorResponseV1(err.Error(), true) + } + + respStruct := models.SetQueueAttributesResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + return http.StatusOK, respStruct +} diff --git a/shared-sqs/app/gosqs/set_queue_attributes_test.go b/shared-sqs/app/gosqs/set_queue_attributes_test.go new file mode 100644 index 0000000..19c730b --- /dev/null +++ b/shared-sqs/app/gosqs/set_queue_attributes_test.go @@ -0,0 +1,169 @@ +package gosqs + +import ( + "fmt" + "net/http" + "testing" + + "shared-sqs/app/test" + + "shared-sqs/app/conf" + "shared-sqs/app/fixtures" + "shared-sqs/app/interfaces" + "shared-sqs/app/models" + "shared-sqs/app/utils" + "github.com/stretchr/testify/assert" +) + +func TestSetQueueAttributesV1_success_multiple_attributes(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SetQueueAttributesRequest) + *v = fixtures.SetQueueAttributesRequest + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := SetQueueAttributesV1(r) + + expectedResponse := models.SetQueueAttributesResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, expectedResponse, response) + + actualQueue := models.SyncQueues.Queues["unit-queue1"] + assert.Equal(t, 5, actualQueue.VisibilityTimeout) + assert.Equal(t, 4, actualQueue.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, 1, actualQueue.DelaySeconds) + assert.Equal(t, 2, actualQueue.MaximumMessageSize) + assert.Equal(t, 3, actualQueue.MessageRetentionPeriod) +} + +func TestSetQueueAttributesV1_success_single_attribute(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SetQueueAttributesRequest) + *v = models.SetQueueAttributesRequest{ + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + Attributes: models.QueueAttributes{ + VisibilityTimeout: 5, + }, + } + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, response := SetQueueAttributesV1(r) + + expectedResponse := models.SetQueueAttributesResponse{ + Xmlns: models.BaseXmlns, + Metadata: models.BaseResponseMetadata, + } + assert.Equal(t, http.StatusOK, code) + assert.Equal(t, expectedResponse, response) + + actualQueue := models.SyncQueues.Queues["unit-queue1"] + assert.Equal(t, 5, actualQueue.VisibilityTimeout) + assert.Equal(t, 0, actualQueue.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, 0, actualQueue.DelaySeconds) + assert.Equal(t, 0, actualQueue.MaximumMessageSize) + assert.Equal(t, 345600, actualQueue.MessageRetentionPeriod) +} + +func TestSetQueueAttributesV1_invalid_request_body(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + return false + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := SetQueueAttributesV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} + +func TestSetQueueAttributesV1_missing_queue_url(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SetQueueAttributesRequest) + *v = models.SetQueueAttributesRequest{ + Attributes: models.QueueAttributes{}, + } + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := SetQueueAttributesV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} + +func TestSetQueueAttributesV1_missing_expected_queue(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SetQueueAttributesRequest) + *v = models.SetQueueAttributesRequest{ + QueueUrl: "garbage", + } + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := SetQueueAttributesV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} + +func TestSetQueueAttributesV1_invalid_redrive_queue(t *testing.T) { + conf.LoadYamlConfig("../conf/mock-data/mock-config.yaml", "BaseUnitTests") + defer func() { + models.ResetApp() + utils.REQUEST_TRANSFORMER = utils.TransformRequest + }() + + utils.REQUEST_TRANSFORMER = func(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + v := resultingStruct.(*models.SetQueueAttributesRequest) + *v = models.SetQueueAttributesRequest{ + QueueUrl: fmt.Sprintf("%s/%s", fixtures.BASE_URL, "unit-queue1"), + Attributes: models.QueueAttributes{ + RedrivePolicy: models.RedrivePolicy{ + MaxReceiveCount: 100, + DeadLetterTargetArn: fmt.Sprintf("arn:aws:sqs:us-east-1:100010001000:%s", "garbage"), + }, + }, + } + return true + } + + _, r := test.GenerateRequestInfo("POST", "/", nil, true) + code, _ := SetQueueAttributesV1(r) + + assert.Equal(t, http.StatusBadRequest, code) +} diff --git a/shared-sqs/app/interfaces/interfaces.go b/shared-sqs/app/interfaces/interfaces.go new file mode 100644 index 0000000..ba4dbd1 --- /dev/null +++ b/shared-sqs/app/interfaces/interfaces.go @@ -0,0 +1,28 @@ +package interfaces + +import ( + "net/url" + + "shared-sqs/app/models" +) + +type AbstractRequestBody interface { + SetAttributesFromForm(values url.Values) +} + +type AbstractResponseBody interface { + GetResult() interface{} + GetRequestId() string +} + +type AbstractErrorResponse interface { + Response() models.ErrorResult + StatusCode() int +} + +type AbstractPublishEntry interface { + GetMessage() string + GetMessageAttributes() map[string]models.MessageAttribute + GetMessageStructure() string + GetSubject() string +} diff --git a/shared-sqs/app/models/configuration.go b/shared-sqs/app/models/configuration.go new file mode 100644 index 0000000..4053973 --- /dev/null +++ b/shared-sqs/app/models/configuration.go @@ -0,0 +1,37 @@ +package models + +/*** config ***/ +type EnvQueue struct { +Name string +ReceiveMessageWaitTimeSeconds int +RedrivePolicy string +MaximumMessageSize int +VisibilityTimeout int +MessageRetentionPeriod int +} + +type EnvQueueAttributes struct { +VisibilityTimeout int +ReceiveMessageWaitTimeSeconds int +MaximumMessageSize int +MessageRetentionPeriod int // seconds +} + +type Environment struct { +Host string +Port string +SqsPort string +Region string +AccountID string +LogToFile bool +LogFile string +EnableDuplicates bool +Queues []EnvQueue +QueueAttributeDefaults EnvQueueAttributes +RandomLatency RandomLatency +} + +type RandomLatency struct { +Min int +Max int +} diff --git a/shared-sqs/app/models/constants.go b/shared-sqs/app/models/constants.go new file mode 100644 index 0000000..1ed7cbf --- /dev/null +++ b/shared-sqs/app/models/constants.go @@ -0,0 +1,38 @@ +package models + +import ( + "time" +) + +var BaseXmlns = "http://queue.amazonaws.com/doc/2012-11-05/" +var BaseResponseMetadata = ResponseMetadata{RequestId: "00000000-0000-0000-0000-000000000000"} + +var DeduplicationPeriod = 5 * time.Minute + +var AvailableQueueAttributes = map[string]bool{ + "DelaySeconds": true, + "MaximumMessageSize": true, + "MessageRetentionPeriod": true, + "Policy": true, + "ReceiveMessageWaitTimeSeconds": true, + "VisibilityTimeout": true, + "RedrivePolicy": true, + "RedriveAllowPolicy": true, + "ApproximateNumberOfMessages": true, + "ApproximateNumberOfMessagesDelayed": true, + "ApproximateNumberOfMessagesNotVisible": true, + "CreatedTimestamp": true, + "LastModifiedTimestamp": true, + "QueueArn": true, +} + +const ( + ProtocolSQS Protocol = "sqs" + ProtocolHTTP Protocol = "http" + ProtocolHTTPS Protocol = "https" + ProtocolDefault Protocol = "default" +) + +const ( + MessageStructureJSON MessageStructure = "json" +) diff --git a/shared-sqs/app/models/conversions.go b/shared-sqs/app/models/conversions.go new file mode 100644 index 0000000..35c7530 --- /dev/null +++ b/shared-sqs/app/models/conversions.go @@ -0,0 +1,36 @@ +package models + +import ( + "encoding/json" + "strconv" +) + +// StringToInt this is a custom type that will allow our request bodies to support either a string OR an int. +// It has its own UnmarshalJSON method to handle both types automatically and it can return an `int` +// from the `Int` method. +type StringToInt int + +func (s *StringToInt) UnmarshalJSON(data []byte) error { + var i int + err := json.Unmarshal(data, &i) + if err == nil { + *s = StringToInt(i) + return nil + } + + var str string + err = json.Unmarshal(data, &str) + if err != nil { + return err + } + tmp, err := strconv.Atoi(str) + if err != nil { + return err + } + *s = StringToInt(tmp) + return nil +} + +func (s *StringToInt) Int() int { + return int(*s) +} diff --git a/shared-sqs/app/models/conversions_test.go b/shared-sqs/app/models/conversions_test.go new file mode 100644 index 0000000..3814dbd --- /dev/null +++ b/shared-sqs/app/models/conversions_test.go @@ -0,0 +1,75 @@ +package models + +import ( + "encoding/json" + "testing" + + "shared-sqs/app/test" + "github.com/stretchr/testify/assert" +) + +type StringToIntStruct struct { + Field1 StringToInt `json:"Field1"` + Field2 StringToInt `json:"Field2"` +} + +func TestStringToInt_unmarshalJSON_int(t *testing.T) { + body := struct { + Field1 int `json:"Field1"` + Field2 int `json:"Field2"` + }{ + Field1: 1, + Field2: 2, + } + _, r := test.GenerateRequestInfo("POST", "/", body, true) + + result := &StringToIntStruct{} + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(result) + + assert.Nil(t, err) + assert.Equal(t, StringToInt(1), result.Field1) + assert.Equal(t, StringToInt(2), result.Field2) +} + +func TestStringToInt_unmarshalJSON_string(t *testing.T) { + body := struct { + Field1 string `json:"Field1"` + Field2 string `json:"Field2"` + }{ + Field1: "1", + Field2: "2", + } + _, r := test.GenerateRequestInfo("POST", "/", body, true) + + result := &StringToIntStruct{} + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(result) + + assert.Nil(t, err) + assert.Equal(t, StringToInt(1), result.Field1) + assert.Equal(t, StringToInt(2), result.Field2) +} + +func TestStringToInt_unmarshalJSON_invalid_type_returns_error(t *testing.T) { + body := struct { + Field1 bool `json:"Field1"` + Field2 bool `json:"Field2"` + }{ + Field1: true, + Field2: false, + } + _, r := test.GenerateRequestInfo("POST", "/", body, true) + + result := &StringToIntStruct{} + decoder := json.NewDecoder(r.Body) + err := decoder.Decode(result) + + assert.Error(t, err) +} + +func TestStringToInt_int_returns_int_type(t *testing.T) { + s := StringToInt(1) + + assert.Equal(t, int(1), s.Int()) +} diff --git a/shared-sqs/app/models/errors.go b/shared-sqs/app/models/errors.go new file mode 100644 index 0000000..5495c7a --- /dev/null +++ b/shared-sqs/app/models/errors.go @@ -0,0 +1,65 @@ +package models + +import "net/http" + +func init() { + SqsErrors = map[string]SqsErrorType{ + "QueueNotFound": {HttpError: http.StatusBadRequest, Type: "Not Found", Code: "AWS.SimpleQueueService.NonExistentQueue", Message: "The specified queue does not exist for this wsdl version."}, + "QueueExists": {HttpError: http.StatusBadRequest, Type: "Duplicate", Code: "AWS.SimpleQueueService.QueueExists", Message: "The specified queue already exists."}, + "MessageDoesNotExist": {HttpError: http.StatusNotFound, Type: "Not Found", Code: "AWS.SimpleQueueService.QueueExists", Message: "The specified queue does not contain the message specified."}, + "GeneralError": {HttpError: http.StatusBadRequest, Type: "GeneralError", Code: "AWS.SimpleQueueService.GeneralError", Message: "General Error."}, + "TooManyEntriesInBatchRequest": {HttpError: http.StatusBadRequest, Type: "TooManyEntriesInBatchRequest", Code: "AWS.SimpleQueueService.TooManyEntriesInBatchRequest", Message: "Maximum number of entries per request are 10."}, + "BatchEntryIdsNotDistinct": {HttpError: http.StatusBadRequest, Type: "BatchEntryIdsNotDistinct", Code: "AWS.SimpleQueueService.BatchEntryIdsNotDistinct", Message: "Two or more batch entries in the request have the same Id."}, + "EmptyBatchRequest": {HttpError: http.StatusBadRequest, Type: "EmptyBatchRequest", Code: "AWS.SimpleQueueService.EmptyBatchRequest", Message: "The batch request doesn't contain any entries."}, + "InvalidVisibilityTimeout": {HttpError: http.StatusBadRequest, Type: "ValidationError", Code: "AWS.SimpleQueueService.ValidationError", Message: "The visibility timeout is incorrect"}, + "MessageNotInFlight": {HttpError: http.StatusBadRequest, Type: "MessageNotInFlight", Code: "AWS.SimpleQueueService.MessageNotInFlight", Message: "The message referred to isn't in flight."}, + "MessageTooBig": {HttpError: http.StatusBadRequest, Type: "MessageTooBig", Code: "InvalidParameterValue", Message: "The message size exceeds the limit."}, + "InvalidParameterValue": {HttpError: http.StatusBadRequest, Type: "InvalidParameterValue", Code: "AWS.SimpleQueueService.InvalidParameterValue", Message: "An invalid or out-of-range value was supplied for the input parameter."}, + "InvalidAttributeValue": {HttpError: http.StatusBadRequest, Type: "InvalidAttributeValue", Code: "AWS.SimpleQueueService.InvalidAttributeValue", Message: "Invalid Value for the parameter RedrivePolicy."}, + } + SnsErrors = map[string]SnsErrorType{ + "InvalidParameterValue": {HttpError: http.StatusBadRequest, Type: "InvalidParameterValue", Code: "AWS.SimpleNotificationService.InvalidParameterValue", Message: "An invalid or out-of-range value was supplied for the input parameter."}, + "TopicNotFound": {HttpError: http.StatusBadRequest, Type: "Not Found", Code: "AWS.SimpleNotificationService.NonExistentTopic", Message: "The specified topic does not exist for this wsdl version."}, + "SubscriptionNotFound": {HttpError: http.StatusNotFound, Type: "Not Found", Code: "AWS.SimpleNotificationService.NonExistentSubscription", Message: "The specified subscription does not exist for this wsdl version."}, + "TopicExists": {HttpError: http.StatusBadRequest, Type: "Duplicate", Code: "AWS.SimpleNotificationService.TopicAlreadyExists", Message: "The specified topic already exists."}, + "ValidationError": {HttpError: http.StatusBadRequest, Type: "InvalidParameter", Code: "AWS.SimpleNotificationService.ValidationError", Message: "The input fails to satisfy the constraints specified by an AWS service."}, + "BatchEntryIdsNotDistinct": {HttpError: http.StatusBadRequest, Type: "BatchEntryIdsNotDistinct", Code: "AWS.SimpleNotificationService.BatchEntryIdsNotDistinct", Message: "Two or more batch entries in the request have the same Id."}, + "EmptyBatchRequest": {HttpError: http.StatusBadRequest, Type: "EmptyBatchRequest", Code: "AWS.SimpleNotificationService.EmptyBatchRequest", Message: "The batch request doesn't contain any entries."}, + "TooManyEntriesInBatchRequest": {HttpError: http.StatusBadRequest, Type: "TooManyEntriesInBatchRequest", Code: "AWS.SimpleNotificationService.TooManyEntriesInBatchRequest", Message: "Maximum number of entries per request are 10."}, + "MalformedInput": {HttpError: http.StatusBadRequest, Type: "Sender", Code: "AWS.SimpleNotificationService.MalformedInput", Message: "Invalid Base64 encoding"}, + } +} + +type SqsErrorType struct { + HttpError int + Type string + Code string + Message string +} + +func (s SqsErrorType) StatusCode() int { + return s.HttpError +} + +func (s SqsErrorType) Response() ErrorResult { + return ErrorResult{Type: s.Type, Code: s.Code, Message: s.Message} +} + +var SqsErrors map[string]SqsErrorType + +type SnsErrorType struct { + HttpError int + Type string + Code string + Message string +} + +func (s SnsErrorType) StatusCode() int { + return s.HttpError +} + +func (s SnsErrorType) Response() ErrorResult { + return ErrorResult{Type: s.Type, Code: s.Code, Message: s.Message} +} + +var SnsErrors map[string]SnsErrorType diff --git a/shared-sqs/app/models/globals.go b/shared-sqs/app/models/globals.go new file mode 100644 index 0000000..8d0c050 --- /dev/null +++ b/shared-sqs/app/models/globals.go @@ -0,0 +1,23 @@ +package models + +import ( + "sync" +) + +// CurrentEnvironment should get overwritten when the app starts up and loads the config. For the +// sake of generating "partial" apps piece-meal during test automation we'll slap these placeholder +// values in here so the resource URLs aren't wonky like `http://://new-queue`. +var CurrentEnvironment = Environment{ + Host: "host", + Port: "port", + Region: "region", + AccountID: "accountID", +} + +var LogMessages bool +var LogFile string + +var SyncQueues = struct { + sync.RWMutex + Queues map[string]*Queue +}{Queues: make(map[string]*Queue)} diff --git a/shared-sqs/app/models/helpers.go b/shared-sqs/app/models/helpers.go new file mode 100644 index 0000000..cf5c91d --- /dev/null +++ b/shared-sqs/app/models/helpers.go @@ -0,0 +1,48 @@ +package models + +import ( + "errors" + "fmt" + "math/rand" + "time" +) + +// ---- Unit Tests ---- +func ResetApp() { + CurrentEnvironment = Environment{} + ResetResources() +} + +func ResetResources() { + SyncQueues.Lock() + SyncQueues.Queues = make(map[string]*Queue) + SyncQueues.Unlock() +} + +func stringInSlice(a string, list []string) bool { + for _, b := range list { + if b == a { + return true + } + } + return false +} + +func generateRandomLatency() (time.Duration, error) { + min := CurrentEnvironment.RandomLatency.Min + max := CurrentEnvironment.RandomLatency.Max + if min == 0 && max == 0 { + return time.Duration(0), nil + } + var randomLatencyValue int + if max == min { + randomLatencyValue = max + } else { + randomLatencyValue = rand.Intn(max-min) + min + } + randomDuration, err := time.ParseDuration(fmt.Sprintf("%dms", randomLatencyValue)) + if err != nil { + return time.Duration(0), errors.New(fmt.Sprintf("Error parsing random latency value: %dms", randomLatencyValue)) + } + return randomDuration, nil +} diff --git a/shared-sqs/app/models/models.go b/shared-sqs/app/models/models.go new file mode 100644 index 0000000..637a4fd --- /dev/null +++ b/shared-sqs/app/models/models.go @@ -0,0 +1,116 @@ +package models + +import ( + "strconv" + "time" + + log "github.com/sirupsen/logrus" +) + +type MessageStructure string +type Protocol string + +type MessageAttribute struct { + BinaryListValues []string `json:"BinaryListValues,omitempty" xml:"BinaryListValues,omitempty"` // currently unsupported by AWS + BinaryValue string `json:"BinaryValue,omitempty" xml:"BinaryValue,omitempty"` + DataType string `json:"DataType,omitempty" xml:"DataType,omitempty"` + StringListValues []string `json:"StringListValues,omitempty" xml:"StringListValues,omitempty"` // currently unsupported by AWS + StringValue string `json:"StringValue,omitempty" xml:"StringValue,omitempty"` +} + + +type SqsMessage struct { + MessageBody string + Uuid string + MD5OfMessageAttributes string + MD5OfMessageBody string + ReceiptHandle string + ReceiptTime time.Time + VisibilityTimeout time.Time + NumberOfReceives int + Retry int + MessageAttributes map[string]MessageAttribute + GroupID string + DeduplicationID string + SentTime time.Time + DelaySecs int +} + +func (m *SqsMessage) IsReadyForReceipt() bool { + randomLatency, err := generateRandomLatency() + if err != nil { + log.Error(err) + return true + } + showAt := m.SentTime.Add(randomLatency).Add(time.Duration(m.DelaySecs) * time.Second) + return showAt.Before(time.Now()) +} + +type Queue struct { + Name string + URL string + Arn string + VisibilityTimeout int // seconds + ReceiveMessageWaitTimeSeconds int + DelaySeconds int + MaximumMessageSize int + MessageRetentionPeriod int // seconds // TODO - not used in the code yet + Messages []SqsMessage + DeadLetterQueue *Queue + MaxReceiveCount int + IsFIFO bool + FIFOMessages map[string]int + FIFOSequenceNumbers map[string]int + EnableDuplicates bool + Duplicates map[string]time.Time +} + +func (q *Queue) NextSequenceNumber(groupId string) string { + if _, ok := q.FIFOSequenceNumbers[groupId]; !ok { + q.FIFOSequenceNumbers = map[string]int{ + groupId: 0, + } + } + + q.FIFOSequenceNumbers[groupId]++ + return strconv.Itoa(q.FIFOSequenceNumbers[groupId]) +} + +func (q *Queue) IsLocked(groupId string) bool { + _, ok := q.FIFOMessages[groupId] + return ok +} + +func (q *Queue) LockGroup(groupId string) { + if _, ok := q.FIFOMessages[groupId]; !ok { + q.FIFOMessages = map[string]int{ + groupId: 0, + } + } +} + +func (q *Queue) UnlockGroup(groupId string) { + if _, ok := q.FIFOMessages[groupId]; ok { + delete(q.FIFOMessages, groupId) + } +} + +func (q *Queue) IsDuplicate(deduplicationId string) bool { + if !q.EnableDuplicates || !q.IsFIFO || deduplicationId == "" { + return false + } + + _, ok := q.Duplicates[deduplicationId] + + return ok +} + +func (q *Queue) InitDuplicatation(deduplicationId string) { + if !q.EnableDuplicates || !q.IsFIFO || deduplicationId == "" { + return + } + + if _, ok := q.Duplicates[deduplicationId]; !ok { + q.Duplicates[deduplicationId] = time.Now() + } +} diff --git a/shared-sqs/app/models/models_test.go b/shared-sqs/app/models/models_test.go new file mode 100644 index 0000000..4954b07 --- /dev/null +++ b/shared-sqs/app/models/models_test.go @@ -0,0 +1,73 @@ +package models + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestFilterPolicy_IsSatisfiedBy(t *testing.T) { + var tests = []struct { + filterPolicy *FilterPolicy + messageAttributes map[string]MessageAttribute + expected bool + }{ + { + &FilterPolicy{"foo": {"bar"}}, + map[string]MessageAttribute{"foo": {DataType: "String", StringValue: "bar"}}, + true, + }, + { + &FilterPolicy{"foo": {"bar", "xyz"}}, + map[string]MessageAttribute{"foo": {DataType: "String", StringValue: "xyz"}}, + true, + }, + { + &FilterPolicy{"foo": {"bar", "xyz"}, "abc": {"def"}}, + map[string]MessageAttribute{"foo": {DataType: "String", StringValue: "xyz"}, + "abc": {DataType: "String", StringValue: "def"}}, + true, + }, + { + &FilterPolicy{"foo": {"bar"}}, + map[string]MessageAttribute{"foo": {DataType: "String", StringValue: "baz"}}, + false, + }, + { + &FilterPolicy{"foo": {"bar"}}, + map[string]MessageAttribute{}, + false, + }, + { + &FilterPolicy{"foo": {"bar"}, "abc": {"def"}}, + map[string]MessageAttribute{"foo": {DataType: "String", StringValue: "bar"}}, + false, + }, + { + &FilterPolicy{"foo": {"bar"}}, + map[string]MessageAttribute{"foo": {DataType: "Binary", BinaryValue: "bar"}}, + false, + }, + } + + for i, tt := range tests { + actual := tt.filterPolicy.IsSatisfiedBy(tt.messageAttributes) + if tt.filterPolicy.IsSatisfiedBy(tt.messageAttributes) != tt.expected { + t.Errorf("#%d FilterPolicy: expected %t, actual %t", i, tt.expected, actual) + } + } + +} + +func TestMessage_IsReadyForReceipt(t *testing.T) { + CurrentEnvironment.RandomLatency.Min = 100 + CurrentEnvironment.RandomLatency.Max = 100 + msg := SqsMessage{ + SentTime: time.Now(), + } + assert.False(t, msg.IsReadyForReceipt()) + duration, _ := time.ParseDuration("105ms") + time.Sleep(duration) + assert.True(t, msg.IsReadyForReceipt()) +} diff --git a/shared-sqs/app/models/requests.go b/shared-sqs/app/models/requests.go new file mode 100644 index 0000000..5049e79 --- /dev/null +++ b/shared-sqs/app/models/requests.go @@ -0,0 +1,515 @@ +package models + +import ( + "encoding/json" + "fmt" + "net/url" + "strconv" + + log "github.com/sirupsen/logrus" +) + +type CreateQueueRequest struct { + QueueName string `json:"QueueName" schema:"QueueName"` + Attributes QueueAttributes `json:"Attributes" schema:"Attribute"` + Tags map[string]string `json:"Tags" schema:"Tags"` + Version string `json:"Version" schema:"Version"` +} + +// TODO - is there an easier way to do this? Similar to the StringToInt type? +func (r *CreateQueueRequest) SetAttributesFromForm(values url.Values) { + for i := 1; true; i++ { + nameKey := fmt.Sprintf("Attribute.%d.Name", i) + attrName := values.Get(nameKey) + if attrName == "" { + break + } + + valueKey := fmt.Sprintf("Attribute.%d.Value", i) + attrValue := values.Get(valueKey) + if attrValue == "" { + continue + } + switch attrName { + case "DelaySeconds": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.DelaySeconds = StringToInt(tmp) + case "MaximumMessageSize": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.MaximumMessageSize = StringToInt(tmp) + case "MessageRetentionPeriod": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.MessageRetentionPeriod = StringToInt(tmp) + case "Policy": + var tmp map[string]interface{} + err := json.Unmarshal([]byte(attrValue), &tmp) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.Policy = tmp + case "ReceiveMessageWaitTimeSeconds": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.ReceiveMessageWaitTimeSeconds = StringToInt(tmp) + case "VisibilityTimeout": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.VisibilityTimeout = StringToInt(tmp) + case "RedrivePolicy": + tmp := RedrivePolicy{} + var decodedPolicy struct { + MaxReceiveCount interface{} `json:"maxReceiveCount"` + DeadLetterTargetArn string `json:"deadLetterTargetArn"` + } + err := json.Unmarshal([]byte(attrValue), &decodedPolicy) + if err != nil || decodedPolicy.DeadLetterTargetArn == "" { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + // Support both int and string types (historic processing), set a default of 10 if not provided. + // Go will default into float64 for interface{} types when parsing numbers + receiveCount, ok := decodedPolicy.MaxReceiveCount.(float64) + if !ok { + receiveCount = 10 + t, ok := decodedPolicy.MaxReceiveCount.(string) + if ok { + r, err := strconv.ParseFloat(t, 64) + if err == nil { + receiveCount = r + } else { + log.Debugf("Failed to parse form attribute (maxReceiveCount) - %s: %s", attrName, attrValue) + } + } else { + log.Debugf("Failed to parse form attribute (maxReceiveCount) - %s: %s", attrName, attrValue) + } + } + tmp.MaxReceiveCount = StringToInt(receiveCount) + tmp.DeadLetterTargetArn = decodedPolicy.DeadLetterTargetArn + r.Attributes.RedrivePolicy = tmp + case "RedriveAllowPolicy": + var tmp map[string]interface{} + err := json.Unmarshal([]byte(attrValue), &tmp) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.RedriveAllowPolicy = tmp + } + } + return +} + +func NewListQueuesRequest() *ListQueueRequest { + return &ListQueueRequest{} +} + +type ListQueueRequest struct { + MaxResults int `json:"MaxResults" schema:"MaxResults"` + NextToken string `json:"NextToken" schema:"NextToken"` + QueueNamePrefix string `json:"QueueNamePrefix" schema:"QueueNamePrefix"` +} + +func (r *ListQueueRequest) SetAttributesFromForm(values url.Values) { + maxResults, err := strconv.Atoi(values.Get("MaxResults")) + if err == nil { + r.MaxResults = maxResults + } + r.NextToken = values.Get("NextToken") + r.QueueNamePrefix = values.Get("QueueNamePrefix") +} + +func NewGetQueueAttributesRequest() *GetQueueAttributesRequest { + return &GetQueueAttributesRequest{} +} + +type GetQueueAttributesRequest struct { + QueueUrl string `json:"QueueUrl"` + AttributeNames []string `json:"AttributeNames"` +} + +func (r *GetQueueAttributesRequest) SetAttributesFromForm(values url.Values) { + r.QueueUrl = values.Get("QueueUrl") + for i := 1; true; i++ { + attrKey := fmt.Sprintf("AttributeName.%d", i) + attrValue := values.Get(attrKey) + if attrValue == "" { + break + } + r.AttributeNames = append(r.AttributeNames, attrValue) + } +} + +/*** Send Message Request */ +func NewSendMessageRequest() *SendMessageRequest { + return &SendMessageRequest{ + MessageAttributes: make(map[string]MessageAttribute), + MessageSystemAttributes: make(map[string]MessageAttribute), + } +} + +type SendMessageRequest struct { + DelaySeconds int `json:"DelaySeconds" schema:"DelaySeconds"` + // MessageAttributes is custom attributes that users can add on the message as they like. + // Please see: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html#SQS-SendMessage-request-MessageAttributes + MessageAttributes map[string]MessageAttribute `json:"MessageAttributes" schema:"MessageAttributes"` + MessageBody string `json:"MessageBody" schema:"MessageBody"` + MessageDeduplicationId string `json:"MessageDeduplicationId" schema:"MessageDeduplicationId"` + MessageGroupId string `json:"MessageGroupId" schema:"MessageGroupId"` + // MessageSystemAttributes is custom attributes for AWS services. + // Please see: https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html#SQS-SendMessage-request-MessageSystemAttributes + // On AWS, the only supported attribute is "AWSTraceHeader" that is for AWS X-Ray. + // Goaws does not contains X-Ray emulation, so currently MessageSystemAttributes is unsupported. + // TODO: Replace with a struct with known attributes "AWSTraceHeader". + MessageSystemAttributes map[string]MessageAttribute `json:"MessageSystemAttributes" schema:"MessageSystemAttributes"` + QueueUrl string `json:"QueueUrl" schema:"QueueUrl"` +} + +func parseMessageAttributes(values url.Values, keyPrefix string) map[string]MessageAttribute { + result := map[string]MessageAttribute{} + + for i := 1; true; i++ { + nameKey := fmt.Sprintf("%s.%d.Name", keyPrefix, i) + name := values.Get(nameKey) + if name == "" { + break + } + + dataTypeKey := fmt.Sprintf("%s.%d.Value.DataType", keyPrefix, i) + dataType := values.Get(dataTypeKey) + if dataType == "" { + log.Warnf("DataType of message attribute %s is missing, MD5 checksum will most probably be wrong!\n", name) + continue + } + + stringValue := values.Get(fmt.Sprintf("%s.%d.Value.StringValue", keyPrefix, i)) + binaryValue := values.Get(fmt.Sprintf("%s.%d.Value.BinaryValue", keyPrefix, i)) + + result[name] = MessageAttribute{ + DataType: dataType, + StringValue: stringValue, + BinaryValue: binaryValue, + } + } + + if len(result) > 0 { + return result + } + return nil +} + +func (r *SendMessageRequest) SetAttributesFromForm(values url.Values) { + r.MessageAttributes = parseMessageAttributes(values, "MessageAttribute") +} + +func NewSendMessageBatchRequest() *SendMessageBatchRequest { + return &SendMessageBatchRequest{} +} + +type SendMessageBatchRequest struct { + Entries []SendMessageBatchRequestEntry + QueueUrl string +} + +func (r *SendMessageBatchRequest) SetAttributesFromForm(values url.Values) { + for entryIndex := range r.Entries { + r.Entries[entryIndex].MessageAttributes = parseMessageAttributes(values, fmt.Sprintf("Entries.%d.MessageAttributes", entryIndex)) + } +} + +type SendMessageBatchRequestEntry struct { + Id string `json:"Id" schema:"Id"` + MessageBody string `json:"MessageBody" schema:"MessageBody"` + DelaySeconds int `json:"DelaySeconds" schema:"DelaySeconds"` // NOTE: not implemented + MessageAttributes map[string]MessageAttribute `json:"MessageAttributes" schema:"MessageAttributes"` + MessageDeduplicationId string `json:"MessageDeduplicationId" schema:"MessageDeduplicationId"` + MessageGroupId string `json:"MessageGroupId" schema:"MessageGroupId"` + MessageSystemAttributes map[string]MessageAttribute `json:"MessageSystemAttributes" schema:"MessageSystemAttributes"` // NOTE: not implemented +} + +// Get Queue Url Request +func NewGetQueueUrlRequest() *GetQueueUrlRequest { + return &GetQueueUrlRequest{} +} + +type GetQueueUrlRequest struct { + QueueName string `json:"QueueName"` + QueueOwnerAWSAccountId string `json:"QueueOwnerAWSAccountId"` // NOTE: not implemented +} + +func (r *GetQueueUrlRequest) SetAttributesFromForm(values url.Values) {} + +func NewSetQueueAttributesRequest() *SetQueueAttributesRequest { + return &SetQueueAttributesRequest{} +} + +type SetQueueAttributesRequest struct { + QueueUrl string `json:"QueueUrl"` + Attributes QueueAttributes `json:"Attributes"` +} + +func (r *SetQueueAttributesRequest) SetAttributesFromForm(values url.Values) { + r.QueueUrl = values.Get("QueueUrl") + // TODO - could we share with CreateQueueRequest? + for i := 1; true; i++ { + nameKey := fmt.Sprintf("Attribute.%d.Name", i) + attrName := values.Get(nameKey) + if attrName == "" { + break + } + + valueKey := fmt.Sprintf("Attribute.%d.Value", i) + attrValue := values.Get(valueKey) + if attrValue == "" { + continue + } + switch attrName { + case "DelaySeconds": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.DelaySeconds = StringToInt(tmp) + case "MaximumMessageSize": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.MaximumMessageSize = StringToInt(tmp) + case "MessageRetentionPeriod": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.MessageRetentionPeriod = StringToInt(tmp) + case "Policy": + var tmp map[string]interface{} + err := json.Unmarshal([]byte(attrValue), &tmp) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.Policy = tmp + case "ReceiveMessageWaitTimeSeconds": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.ReceiveMessageWaitTimeSeconds = StringToInt(tmp) + case "VisibilityTimeout": + tmp, err := strconv.Atoi(attrValue) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.VisibilityTimeout = StringToInt(tmp) + case "RedrivePolicy": + tmp := RedrivePolicy{} + var decodedPolicy struct { + MaxReceiveCount interface{} `json:"maxReceiveCount"` + DeadLetterTargetArn string `json:"deadLetterTargetArn"` + } + err := json.Unmarshal([]byte(attrValue), &decodedPolicy) + if err != nil || decodedPolicy.DeadLetterTargetArn == "" { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + // Support both int and string types (historic processing), set a default of 10 if not provided. + // Go will default into float64 for interface{} types when parsing numbers + receiveCount, ok := decodedPolicy.MaxReceiveCount.(float64) + if !ok { + receiveCount = 10 + t, ok := decodedPolicy.MaxReceiveCount.(string) + if ok { + r, err := strconv.ParseFloat(t, 64) + if err == nil { + receiveCount = r + } else { + log.Debugf("Failed to parse form attribute (maxReceiveCount) - %s: %s", attrName, attrValue) + } + } else { + log.Debugf("Failed to parse form attribute (maxReceiveCount) - %s: %s", attrName, attrValue) + } + } + tmp.MaxReceiveCount = StringToInt(receiveCount) + tmp.DeadLetterTargetArn = decodedPolicy.DeadLetterTargetArn + r.Attributes.RedrivePolicy = tmp + case "RedriveAllowPolicy": + var tmp map[string]interface{} + err := json.Unmarshal([]byte(attrValue), &tmp) + if err != nil { + log.Debugf("Failed to parse form attribute - %s: %s", attrName, attrValue) + continue + } + r.Attributes.RedriveAllowPolicy = tmp + } + } + return +} + +// TODO - there are FIFO attributes and things too +// QueueAttributes - SQS QueueAttributes Available in create/set attributes requests. +// https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_CreateQueue.html#SQS-CreateQueue-request-attributes +type QueueAttributes struct { + DelaySeconds StringToInt `json:"DelaySeconds"` + MaximumMessageSize StringToInt `json:"MaximumMessageSize"` + MessageRetentionPeriod StringToInt `json:"MessageRetentionPeriod"` // NOTE: not implemented + Policy map[string]interface{} `json:"Policy"` // NOTE: not implemented + ReceiveMessageWaitTimeSeconds StringToInt `json:"ReceiveMessageWaitTimeSeconds"` + VisibilityTimeout StringToInt `json:"VisibilityTimeout"` + // Dead Letter Queues Only + RedrivePolicy RedrivePolicy `json:"RedrivePolicy"` + RedriveAllowPolicy map[string]interface{} `json:"RedriveAllowPolicy"` // NOTE: not implemented +} + +type RedrivePolicy struct { + MaxReceiveCount StringToInt `json:"maxReceiveCount"` + DeadLetterTargetArn string `json:"deadLetterTargetArn"` +} + +// UnmarshalJSON this will convert a JSON string of a Redrive Policy sub-doc (escaped characters and all) or +// a regular json document into the appropriate resulting struct. +func (r *RedrivePolicy) UnmarshalJSON(data []byte) error { + type basicRequest RedrivePolicy + + err := json.Unmarshal(data, (*basicRequest)(r)) + if err == nil { + return nil + } + + tmp, _ := strconv.Unquote(string(data)) + err = json.Unmarshal([]byte(tmp), (*basicRequest)(r)) + if err != nil { + return err + } + return nil +} + +func NewReceiveMessageRequest() *ReceiveMessageRequest { + return &ReceiveMessageRequest{} +} + +type ReceiveMessageRequest struct { + QueueUrl string `json:"QueueUrl" schema:"QueueUrl"` + AttributeNames []string `json:"AttributeNames" schema:"AttributeNames"` + MessageSystemAttributeNames []string `json:"MessageSystemAttributeNames" schema:"MessageSystemAttributeNames"` + MessageAttributeNames []string `json:"MessageAttributeNames" schema:"MessageAttributeNames"` + MaxNumberOfMessages int `json:"MaxNumberOfMessages" schema:"MaxNumberOfMessages"` + VisibilityTimeout int `json:"VisibilityTimeout" schema:"VisibilityTimeout"` + WaitTimeSeconds int `json:"WaitTimeSeconds" schema:"WaitTimeSeconds"` + ReceiveRequestAttemptId string `json:"ReceiveRequestAttemptId" schema:"ReceiveRequestAttemptId"` +} + +func (r *ReceiveMessageRequest) SetAttributesFromForm(values url.Values) {} + +func NewCreateQueueRequest() *CreateQueueRequest { + return &CreateQueueRequest{ + Attributes: QueueAttributes{ + DelaySeconds: 0, + MaximumMessageSize: StringToInt(CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize), + MessageRetentionPeriod: StringToInt(CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod), + ReceiveMessageWaitTimeSeconds: StringToInt(CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds), + VisibilityTimeout: StringToInt(CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout), + }, + } +} + +func NewChangeMessageVisibilityRequest() *ChangeMessageVisibilityRequest { + return &ChangeMessageVisibilityRequest{} +} + +type ChangeMessageVisibilityRequest struct { + QueueUrl string `json:"QueueUrl" schema:"QueueUrl"` + ReceiptHandle string `json:"ReceiptHandle" schema:"ReceiptHandle"` + VisibilityTimeout int `json:"VisibilityTimeout" schema:"VisibilityTimeout"` +} + +func (r *ChangeMessageVisibilityRequest) SetAttributesFromForm(values url.Values) {} + +func NewDeleteMessageRequest() *DeleteMessageRequest { + return &DeleteMessageRequest{} +} + +type DeleteMessageRequest struct { + QueueUrl string `json:"QueueUrl" schema:"QueueUrl"` + ReceiptHandle string `json:"ReceiptHandle" schema:"ReceiptHandle"` +} + +func (r *DeleteMessageRequest) SetAttributesFromForm(values url.Values) {} + +func NewPurgeQueueRequest() *PurgeQueueRequest { + return &PurgeQueueRequest{} +} + +type PurgeQueueRequest struct { + QueueUrl string `json:"QueueUrl" schema:"QueueUrl"` +} + +func (r *PurgeQueueRequest) SetAttributesFromForm(values url.Values) {} + +func NewDeleteQueueRequest() *DeleteQueueRequest { + return &DeleteQueueRequest{} +} + +type DeleteQueueRequest struct { + QueueUrl string `json:"QueueUrl" schema:"QueueUrl"` +} + +func (r *DeleteQueueRequest) SetAttributesFromForm(values url.Values) {} + +type DeleteMessageBatchRequestEntry struct { + Id string `json:"Id" schema:"Id"` + ReceiptHandle string `json:"ReceiptHandle" schema:"ReceiptHandle"` +} + +type DeleteMessageBatchRequest struct { + Entries []DeleteMessageBatchRequestEntry `json:"Entries"` + QueueUrl string `json:"QueueUrl" schema:"QueueUrl"` +} + +func NewDeleteMessageBatchRequest() *DeleteMessageBatchRequest { + return &DeleteMessageBatchRequest{} +} + +func (r *DeleteMessageBatchRequest) SetAttributesFromForm(values url.Values) { + entries := []DeleteMessageBatchRequestEntry{} + for i := 1; true; i++ { + msgIdKey := fmt.Sprintf("DeleteMessageBatchRequestEntry.%d.Id", i) + receiptHandleKey := fmt.Sprintf("DeleteMessageBatchRequestEntry.%d.ReceiptHandle", i) + + msgId := values.Get(msgIdKey) + receiptHandle := values.Get(receiptHandleKey) + if msgId == "" || receiptHandle == "" { + break + } + entries = append(entries, DeleteMessageBatchRequestEntry{ + Id: msgId, + ReceiptHandle: receiptHandle, + }) + } + if len(entries) > 0 { + r.Entries = entries + } +} diff --git a/shared-sqs/app/models/requests_test.go b/shared-sqs/app/models/requests_test.go new file mode 100644 index 0000000..9da5849 --- /dev/null +++ b/shared-sqs/app/models/requests_test.go @@ -0,0 +1,739 @@ +package models + +import ( + "encoding/json" + "fmt" + "math/rand" + "net/url" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestNewCreateQueueRequest(t *testing.T) { + CurrentEnvironment.QueueAttributeDefaults.MaximumMessageSize = 262144 + CurrentEnvironment.QueueAttributeDefaults.MessageRetentionPeriod = 345600 + CurrentEnvironment.QueueAttributeDefaults.ReceiveMessageWaitTimeSeconds = 10 + CurrentEnvironment.QueueAttributeDefaults.VisibilityTimeout = 30 + defer func() { + ResetApp() + }() + + expectedCreateQueueRequest := &CreateQueueRequest{ + Attributes: QueueAttributes{ + DelaySeconds: 0, + MaximumMessageSize: 262144, + MessageRetentionPeriod: 345600, + ReceiveMessageWaitTimeSeconds: 10, + VisibilityTimeout: 30, + }, + } + + result := NewCreateQueueRequest() + + assert.Equal(t, expectedCreateQueueRequest, result) +} + +func TestCreateQueueRequest_SetAttributesFromForm_success(t *testing.T) { + expectedRedrivePolicy := RedrivePolicy{ + MaxReceiveCount: 100, + DeadLetterTargetArn: "dead-letter-queue-arn", + } + + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "new-queue") + form.Add("Version", "2012-11-05") + form.Add("Attribute.1.Name", "DelaySeconds") + form.Add("Attribute.1.Value", "1") + form.Add("Attribute.2.Name", "MaximumMessageSize") + form.Add("Attribute.2.Value", "2") + form.Add("Attribute.3.Name", "MessageRetentionPeriod") + form.Add("Attribute.3.Value", "3") + form.Add("Attribute.4.Name", "Policy") + form.Add("Attribute.4.Value", "{\"i-am\":\"the-policy\"}") + form.Add("Attribute.5.Name", "ReceiveMessageWaitTimeSeconds") + form.Add("Attribute.5.Value", "4") + form.Add("Attribute.6.Name", "VisibilityTimeout") + form.Add("Attribute.6.Value", "5") + form.Add("Attribute.7.Name", "RedrivePolicy") + form.Add("Attribute.7.Value", "{\"maxReceiveCount\": 100, \"deadLetterTargetArn\":\"dead-letter-queue-arn\"}") + form.Add("Attribute.8.Name", "RedriveAllowPolicy") + form.Add("Attribute.8.Value", "{\"i-am\":\"the-redrive-allow-policy\"}") + + cqr := &CreateQueueRequest{ + Attributes: QueueAttributes{ + DelaySeconds: 1, + MaximumMessageSize: 262144, + MessageRetentionPeriod: 345600, + ReceiveMessageWaitTimeSeconds: 10, + VisibilityTimeout: 30, + }, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, StringToInt(1), cqr.Attributes.DelaySeconds) + assert.Equal(t, StringToInt(2), cqr.Attributes.MaximumMessageSize) + assert.Equal(t, StringToInt(3), cqr.Attributes.MessageRetentionPeriod) + assert.Equal(t, map[string]interface{}{"i-am": "the-policy"}, cqr.Attributes.Policy) + assert.Equal(t, StringToInt(4), cqr.Attributes.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, StringToInt(5), cqr.Attributes.VisibilityTimeout) + assert.Equal(t, expectedRedrivePolicy, cqr.Attributes.RedrivePolicy) + assert.Equal(t, map[string]interface{}{"i-am": "the-redrive-allow-policy"}, cqr.Attributes.RedriveAllowPolicy) +} + +func TestCreateQueueRequest_SetAttributesFromForm_success_handles_redrive_recieve_count_int(t *testing.T) { + expectedRedrivePolicy := RedrivePolicy{ + MaxReceiveCount: 100, + DeadLetterTargetArn: "dead-letter-queue-arn", + } + + form := url.Values{} + form.Add("Attribute.1.Name", "RedrivePolicy") + form.Add("Attribute.1.Value", "{\"maxReceiveCount\": 100, \"deadLetterTargetArn\":\"dead-letter-queue-arn\"}") + + cqr := &CreateQueueRequest{ + Attributes: QueueAttributes{}, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, expectedRedrivePolicy, cqr.Attributes.RedrivePolicy) +} + +func TestCreateQueueRequest_SetAttributesFromForm_success_handles_redrive_recieve_count_string(t *testing.T) { + expectedRedrivePolicy := RedrivePolicy{ + MaxReceiveCount: 100, + DeadLetterTargetArn: "dead-letter-queue-arn", + } + + form := url.Values{} + form.Add("Attribute.1.Name", "RedrivePolicy") + form.Add("Attribute.1.Value", "{\"maxReceiveCount\": \"100\", \"deadLetterTargetArn\":\"dead-letter-queue-arn\"}") + + cqr := &CreateQueueRequest{ + Attributes: QueueAttributes{}, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, expectedRedrivePolicy, cqr.Attributes.RedrivePolicy) +} + +func TestCreateQueueRequest_SetAttributesFromForm_success_default_unparsable_redrive_recieve_count(t *testing.T) { + defaultRedrivePolicy := RedrivePolicy{ + MaxReceiveCount: 10, + DeadLetterTargetArn: "dead-letter-queue-arn", + } + + form := url.Values{} + form.Add("Attribute.1.Name", "RedrivePolicy") + form.Add("Attribute.1.Value", "{\"maxReceiveCount\": null, \"deadLetterTargetArn\":\"dead-letter-queue-arn\"}") + + cqr := &CreateQueueRequest{ + Attributes: QueueAttributes{}, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, defaultRedrivePolicy, cqr.Attributes.RedrivePolicy) +} + +func TestCreateQueueRequest_SetAttributesFromForm_success_skips_invalid_values(t *testing.T) { + form := url.Values{} + form.Add("Attribute.1.Name", "DelaySeconds") + form.Add("Attribute.1.Value", "garbage") + form.Add("Attribute.2.Name", "MaximumMessageSize") + form.Add("Attribute.2.Value", "garbage") + form.Add("Attribute.3.Name", "MessageRetentionPeriod") + form.Add("Attribute.3.Value", "garbage") + form.Add("Attribute.4.Name", "Policy") + form.Add("Attribute.4.Value", "garbage") + form.Add("Attribute.5.Name", "ReceiveMessageWaitTimeSeconds") + form.Add("Attribute.5.Value", "garbage") + form.Add("Attribute.6.Name", "VisibilityTimeout") + form.Add("Attribute.6.Value", "garbage") + form.Add("Attribute.7.Name", "RedrivePolicy") + form.Add("Attribute.7.Value", "garbage") + form.Add("Attribute.8.Name", "RedriveAllowPolicy") + form.Add("Attribute.8.Value", "garbage") + + cqr := &CreateQueueRequest{ + Attributes: QueueAttributes{ + DelaySeconds: 1, + MaximumMessageSize: 262144, + MessageRetentionPeriod: 345600, + ReceiveMessageWaitTimeSeconds: 10, + VisibilityTimeout: 30, + }, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, StringToInt(1), cqr.Attributes.DelaySeconds) + assert.Equal(t, StringToInt(262144), cqr.Attributes.MaximumMessageSize) + assert.Equal(t, StringToInt(345600), cqr.Attributes.MessageRetentionPeriod) + assert.Equal(t, map[string]interface{}(nil), cqr.Attributes.Policy) + assert.Equal(t, StringToInt(10), cqr.Attributes.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, StringToInt(30), cqr.Attributes.VisibilityTimeout) + assert.Equal(t, RedrivePolicy{}, cqr.Attributes.RedrivePolicy) + assert.Equal(t, map[string]interface{}(nil), cqr.Attributes.RedriveAllowPolicy) +} + +func TestRedrivePolicy_UnmarshalJSON_handles_nested_json(t *testing.T) { + request := struct { + MaxReceiveCount int `json:"maxReceiveCount"` + DeadLetterTargetArn string `json:"deadLetterTargetArn"` + }{ + MaxReceiveCount: 100, + DeadLetterTargetArn: "arn:redrive-queue", + } + b, _ := json.Marshal(request) + var r = RedrivePolicy{} + err := r.UnmarshalJSON(b) + + assert.Nil(t, err) + assert.Equal(t, StringToInt(100), r.MaxReceiveCount) + assert.Equal(t, fmt.Sprintf("%s:%s", "arn", "redrive-queue"), r.DeadLetterTargetArn) +} + +func TestRedrivePolicy_UnmarshalJSON_handles_escaped_string(t *testing.T) { + request := `{"maxReceiveCount":"100","deadLetterTargetArn":"arn:redrive-queue"}` + b, _ := json.Marshal(request) + var r = RedrivePolicy{} + err := r.UnmarshalJSON(b) + + assert.Nil(t, err) + assert.Equal(t, StringToInt(100), r.MaxReceiveCount) + assert.Equal(t, fmt.Sprintf("%s:%s", "arn", "redrive-queue"), r.DeadLetterTargetArn) +} + +func TestRedrivePolicy_UnmarshalJSON_invalid_json_request_returns_error(t *testing.T) { + request := fmt.Sprintf(`{\"maxReceiveCount\":\"100\",\"deadLetterTargetArn\":\"arn:redrive-queue\"}`) + var r = RedrivePolicy{} + err := r.UnmarshalJSON([]byte(request)) + + assert.Error(t, err) + assert.Equal(t, StringToInt(0), r.MaxReceiveCount) + assert.Equal(t, "", r.DeadLetterTargetArn) +} + +func TestRedrivePolicy_UnmarshalJSON_invalid_type_returns_error(t *testing.T) { + request := `{"maxReceiveCount":true,"deadLetterTargetArn":"arn:redrive-queue"}` + b, _ := json.Marshal(request) + var r = RedrivePolicy{} + err := r.UnmarshalJSON(b) + + assert.Error(t, err) + assert.Equal(t, StringToInt(0), r.MaxReceiveCount) + assert.Equal(t, "", r.DeadLetterTargetArn) +} + +func TestNewListQueuesRequest_SetAttributesFromForm(t *testing.T) { + form := url.Values{} + form.Add("MaxResults", "1") + form.Add("NextToken", "next-token") + form.Add("QueueNamePrefix", "queue-name-prefix") + + lqr := &ListQueueRequest{} + lqr.SetAttributesFromForm(form) + + assert.Equal(t, 1, lqr.MaxResults) + assert.Equal(t, "next-token", lqr.NextToken) + assert.Equal(t, "queue-name-prefix", lqr.QueueNamePrefix) +} + +func TestListQueuesRequest_SetAttributesFromForm_invalid_max_results(t *testing.T) { + form := url.Values{} + form.Add("MaxResults", "1.0") + form.Add("NextToken", "next-token") + form.Add("QueueNamePrefix", "queue-name-prefix") + + lqr := &ListQueueRequest{} + lqr.SetAttributesFromForm(form) + + assert.Equal(t, 0, lqr.MaxResults) + assert.Equal(t, "next-token", lqr.NextToken) + assert.Equal(t, "queue-name-prefix", lqr.QueueNamePrefix) +} + +func TestGetQueueAttributesRequest_SetAttributesFromForm(t *testing.T) { + form := url.Values{} + form.Add("QueueUrl", "queue-url") + form.Add("AttributeName.1", "attribute-1") + form.Add("AttributeName.2", "attribute-2") + + lqr := &GetQueueAttributesRequest{} + lqr.SetAttributesFromForm(form) + + assert.Equal(t, "queue-url", lqr.QueueUrl) + assert.Equal(t, 2, len(lqr.AttributeNames)) + assert.Contains(t, lqr.AttributeNames, "attribute-1") + assert.Contains(t, lqr.AttributeNames, "attribute-2") +} + +func TestGetQueueAttributesRequest_SetAttributesFromForm_skips_invalid_key_sequence(t *testing.T) { + form := url.Values{} + form.Add("QueueUrl", "queue-url") + form.Add("AttributeName.1", "attribute-1") + form.Add("AttributeName.3", "attribute-3") + + lqr := &GetQueueAttributesRequest{} + lqr.SetAttributesFromForm(form) + + assert.Equal(t, "queue-url", lqr.QueueUrl) + assert.Equal(t, 1, len(lqr.AttributeNames)) + assert.Contains(t, lqr.AttributeNames, "attribute-1") +} + +func TestSendMessageRequest_SetAttributesFromForm_success(t *testing.T) { + form := url.Values{} + form.Add("MessageAttribute.1.Name", "Attr1") + form.Add("MessageAttribute.1.Value.DataType", "String") + form.Add("MessageAttribute.1.Value.StringValue", "Value1") + form.Add("MessageAttribute.2.Name", "Attr2") + form.Add("MessageAttribute.2.Value.DataType", "Binary") + form.Add("MessageAttribute.2.Value.BinaryValue", "VmFsdWUy") + form.Add("MessageAttribute.3.Name", "") + form.Add("MessageAttribute.3.Value.DataType", "String") + form.Add("MessageAttribute.3.Value.StringValue", "Value") + form.Add("MessageAttribute.4.Name", "Attr4") + form.Add("MessageAttribute.4.Value.DataType", "") + form.Add("MessageAttribute.4.Value.StringValue", "Value4") + + r := &SendMessageRequest{ + MessageAttributes: make(map[string]MessageAttribute), + MessageSystemAttributes: make(map[string]MessageAttribute), + } + r.SetAttributesFromForm(form) + + assert.Equal(t, 2, len(r.MessageAttributes)) + + assert.NotNil(t, r.MessageAttributes["Attr1"]) + attr1 := r.MessageAttributes["Attr1"] + assert.Equal(t, "String", attr1.DataType) + assert.Equal(t, "Value1", attr1.StringValue) + assert.Empty(t, attr1.BinaryValue) + + assert.NotNil(t, r.MessageAttributes["Attr2"]) + attr2 := r.MessageAttributes["Attr2"] + assert.Equal(t, "Binary", attr2.DataType) + assert.Empty(t, attr2.StringValue) + assert.Equal(t, "VmFsdWUy", attr2.BinaryValue) +} + +func TestSetQueueAttributesRequest_SetAttributesFromForm_success(t *testing.T) { + expectedRedrivePolicy := RedrivePolicy{ + MaxReceiveCount: 100, + DeadLetterTargetArn: "dead-letter-queue-arn", + } + + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "new-queue") + form.Add("Version", "2012-11-05") + form.Add("Attribute.1.Name", "DelaySeconds") + form.Add("Attribute.1.Value", "1") + form.Add("Attribute.2.Name", "MaximumMessageSize") + form.Add("Attribute.2.Value", "2") + form.Add("Attribute.3.Name", "MessageRetentionPeriod") + form.Add("Attribute.3.Value", "3") + form.Add("Attribute.4.Name", "Policy") + form.Add("Attribute.4.Value", "{\"i-am\":\"the-policy\"}") + form.Add("Attribute.5.Name", "ReceiveMessageWaitTimeSeconds") + form.Add("Attribute.5.Value", "4") + form.Add("Attribute.6.Name", "VisibilityTimeout") + form.Add("Attribute.6.Value", "5") + form.Add("Attribute.7.Name", "RedrivePolicy") + form.Add("Attribute.7.Value", "{\"maxReceiveCount\": 100, \"deadLetterTargetArn\":\"dead-letter-queue-arn\"}") + form.Add("Attribute.8.Name", "RedriveAllowPolicy") + form.Add("Attribute.8.Value", "{\"i-am\":\"the-redrive-allow-policy\"}") + + cqr := &SetQueueAttributesRequest{ + Attributes: QueueAttributes{ + DelaySeconds: 1, + MaximumMessageSize: 262144, + MessageRetentionPeriod: 345600, + ReceiveMessageWaitTimeSeconds: 10, + VisibilityTimeout: 30, + }, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, StringToInt(1), cqr.Attributes.DelaySeconds) + assert.Equal(t, StringToInt(2), cqr.Attributes.MaximumMessageSize) + assert.Equal(t, StringToInt(3), cqr.Attributes.MessageRetentionPeriod) + assert.Equal(t, map[string]interface{}{"i-am": "the-policy"}, cqr.Attributes.Policy) + assert.Equal(t, StringToInt(4), cqr.Attributes.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, StringToInt(5), cqr.Attributes.VisibilityTimeout) + assert.Equal(t, expectedRedrivePolicy, cqr.Attributes.RedrivePolicy) + assert.Equal(t, map[string]interface{}{"i-am": "the-redrive-allow-policy"}, cqr.Attributes.RedriveAllowPolicy) +} + +func TestSetQueueAttributesRequest_SetAttributesFromForm_success_handles_redrive_recieve_count_int(t *testing.T) { + expectedRedrivePolicy := RedrivePolicy{ + MaxReceiveCount: 100, + DeadLetterTargetArn: "dead-letter-queue-arn", + } + + form := url.Values{} + form.Add("Attribute.1.Name", "RedrivePolicy") + form.Add("Attribute.1.Value", "{\"maxReceiveCount\": 100, \"deadLetterTargetArn\":\"dead-letter-queue-arn\"}") + + cqr := &SetQueueAttributesRequest{ + Attributes: QueueAttributes{}, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, expectedRedrivePolicy, cqr.Attributes.RedrivePolicy) +} + +func TestSetQueueAttributesRequest_SetAttributesFromForm_success_handles_redrive_recieve_count_string(t *testing.T) { + expectedRedrivePolicy := RedrivePolicy{ + MaxReceiveCount: 100, + DeadLetterTargetArn: "dead-letter-queue-arn", + } + + form := url.Values{} + form.Add("Attribute.1.Name", "RedrivePolicy") + form.Add("Attribute.1.Value", "{\"maxReceiveCount\": \"100\", \"deadLetterTargetArn\":\"dead-letter-queue-arn\"}") + + cqr := &SetQueueAttributesRequest{ + Attributes: QueueAttributes{}, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, expectedRedrivePolicy, cqr.Attributes.RedrivePolicy) +} + +func TestSetQueueAttributesRequest_SetAttributesFromForm_success_default_unparsable_redrive_recieve_count(t *testing.T) { + defaultRedrivePolicy := RedrivePolicy{ + MaxReceiveCount: 10, + DeadLetterTargetArn: "dead-letter-queue-arn", + } + + form := url.Values{} + form.Add("Attribute.1.Name", "RedrivePolicy") + form.Add("Attribute.1.Value", "{\"maxReceiveCount\": null, \"deadLetterTargetArn\":\"dead-letter-queue-arn\"}") + + cqr := &SetQueueAttributesRequest{ + Attributes: QueueAttributes{}, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, defaultRedrivePolicy, cqr.Attributes.RedrivePolicy) +} + +func TestSetQueueAttributesRequest_SetAttributesFromForm_success_skips_invalid_values(t *testing.T) { + form := url.Values{} + form.Add("Attribute.1.Name", "DelaySeconds") + form.Add("Attribute.1.Value", "garbage") + form.Add("Attribute.2.Name", "MaximumMessageSize") + form.Add("Attribute.2.Value", "garbage") + form.Add("Attribute.3.Name", "MessageRetentionPeriod") + form.Add("Attribute.3.Value", "garbage") + form.Add("Attribute.4.Name", "Policy") + form.Add("Attribute.4.Value", "garbage") + form.Add("Attribute.5.Name", "ReceiveMessageWaitTimeSeconds") + form.Add("Attribute.5.Value", "garbage") + form.Add("Attribute.6.Name", "VisibilityTimeout") + form.Add("Attribute.6.Value", "garbage") + form.Add("Attribute.7.Name", "RedrivePolicy") + form.Add("Attribute.7.Value", "garbage") + form.Add("Attribute.8.Name", "RedriveAllowPolicy") + form.Add("Attribute.8.Value", "garbage") + + cqr := &SetQueueAttributesRequest{ + Attributes: QueueAttributes{ + DelaySeconds: 1, + MaximumMessageSize: 262144, + MessageRetentionPeriod: 345600, + ReceiveMessageWaitTimeSeconds: 10, + VisibilityTimeout: 30, + }, + } + cqr.SetAttributesFromForm(form) + + assert.Equal(t, StringToInt(1), cqr.Attributes.DelaySeconds) + assert.Equal(t, StringToInt(262144), cqr.Attributes.MaximumMessageSize) + assert.Equal(t, StringToInt(345600), cqr.Attributes.MessageRetentionPeriod) + assert.Equal(t, map[string]interface{}(nil), cqr.Attributes.Policy) + assert.Equal(t, StringToInt(10), cqr.Attributes.ReceiveMessageWaitTimeSeconds) + assert.Equal(t, StringToInt(30), cqr.Attributes.VisibilityTimeout) + assert.Equal(t, RedrivePolicy{}, cqr.Attributes.RedrivePolicy) + assert.Equal(t, map[string]interface{}(nil), cqr.Attributes.RedriveAllowPolicy) +} + +func TestNewCreateTopicRequest(t *testing.T) { + defer func() { + ResetApp() + }() + + result := NewCreateTopicRequest() + + assert.Equal(t, false, result.Attributes.FifoTopic) + assert.Equal(t, StringToInt(1), result.Attributes.SignatureVersion) + assert.Equal(t, "Active", result.Attributes.TracingConfig) + assert.Equal(t, false, result.Attributes.ContentBasedDeduplication) +} + +func TestCreateTopicRequest_SetAttributesFromForm_success(t *testing.T) { + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "new-queue") + form.Add("Version", "2012-11-05") + form.Add("Attribute.1.Name", "DeliveryPolicy") + form.Add("Attribute.1.Value", "{\"i-am\":\"the-policy\", \"name\":\"delivery-policy\"}") + form.Add("Attribute.2.Name", "DisplayName") + form.Add("Attribute.2.Value", "Foo") + form.Add("Attribute.3.Name", "FifoTopic") + form.Add("Attribute.3.Value", "true") + form.Add("Attribute.4.Name", "Policy") + form.Add("Attribute.4.Value", "{\"i-am\":\"the-policy\", \"name\":\"policy\"}") + form.Add("Attribute.5.Name", "SignatureVersion") + form.Add("Attribute.5.Value", "99") + form.Add("Attribute.6.Name", "TracingConfig") + form.Add("Attribute.6.Value", "PassThrough") + form.Add("Attribute.7.Name", "KmsMasterKeyId") + form.Add("Attribute.7.Value", "1234abcd-12ab-34cd-56ef-1234567890ab") + form.Add("Attribute.8.Name", "ArchivePolicy") + form.Add("Attribute.8.Value", "{\"i-am\":\"the-policy\", \"name\":\"archive-policy\"}") + form.Add("Attribute.9.Name", "BeginningArchiveTime") + form.Add("Attribute.9.Value", "2024-07-01T23:59:59+09:00") + form.Add("Attribute.10.Name", "ContentBasedDeduplication") + form.Add("Attribute.10.Value", "true") + + ctr := &CreateTopicRequest{} + ctr.SetAttributesFromForm(form) + + assert.Equal(t, 2, len(ctr.Attributes.DeliveryPolicy)) + assert.Equal(t, "the-policy", ctr.Attributes.DeliveryPolicy["i-am"]) + assert.Equal(t, "delivery-policy", ctr.Attributes.DeliveryPolicy["name"]) + assert.Equal(t, "Foo", ctr.Attributes.DisplayName) + assert.Equal(t, true, ctr.Attributes.FifoTopic) + assert.Equal(t, 2, len(ctr.Attributes.Policy)) + assert.Equal(t, "the-policy", ctr.Attributes.Policy["i-am"]) + assert.Equal(t, "policy", ctr.Attributes.Policy["name"]) + assert.Equal(t, StringToInt(99), ctr.Attributes.SignatureVersion) + assert.Equal(t, "PassThrough", ctr.Attributes.TracingConfig) + assert.Equal(t, "1234abcd-12ab-34cd-56ef-1234567890ab", ctr.Attributes.KmsMasterKeyId) + assert.Equal(t, 2, len(ctr.Attributes.ArchivePolicy)) + assert.Equal(t, "the-policy", ctr.Attributes.ArchivePolicy["i-am"]) + assert.Equal(t, "archive-policy", ctr.Attributes.ArchivePolicy["name"]) + assert.Equal(t, "2024-07-01T23:59:59+09:00", ctr.Attributes.BeginningArchiveTime) + assert.Equal(t, true, ctr.Attributes.ContentBasedDeduplication) +} + +func TestSubscribeRequest_SetAttributesFromForm_success(t *testing.T) { + form := url.Values{} + form.Add("Attributes.entry.1.key", "RawMessageDelivery") + form.Add("Attributes.entry.1.value", "true") + form.Add("Attributes.entry.2.key", "FilterPolicy") + form.Add("Attributes.entry.2.value", "{\"filter\": [\"policy\"]}") + + cqr := &SubscribeRequest{ + Attributes: SubscriptionAttributes{}, + } + cqr.SetAttributesFromForm(form) + + assert.True(t, cqr.Attributes.RawMessageDelivery) + assert.Equal(t, FilterPolicy{"filter": []string{"policy"}}, cqr.Attributes.FilterPolicy) +} + +func TestSubscribeRequest_SetAttributesFromForm_skips_invalid_values(t *testing.T) { + form := url.Values{} + form.Add("Attributes.entry.1.key", "RawMessageDelivery") + form.Add("Attributes.entry.1.value", "garbage") + form.Add("Attributes.entry.2.key", "FilterPolicy") + form.Add("Attributes.entry.2.value", "also-garbage") + + cqr := &SubscribeRequest{ + Attributes: SubscriptionAttributes{}, + } + cqr.SetAttributesFromForm(form) + + assert.False(t, cqr.Attributes.RawMessageDelivery) + assert.Equal(t, FilterPolicy(nil), cqr.Attributes.FilterPolicy) +} + +func TestSubscribeRequest_SetAttributesFromForm_stops_if_attributes_not_numbered_sequentially(t *testing.T) { + form := url.Values{} + form.Add("Attributes.entry.2.key", "RawMessageDelivery") + form.Add("Attributes.entry.2.value", "garbage") + form.Add("Attributes.entry.3.key", "FilterPolicy") + form.Add("Attributes.entry.3.value", "also-garbage") + + cqr := &SubscribeRequest{ + Attributes: SubscriptionAttributes{}, + } + cqr.SetAttributesFromForm(form) + + assert.False(t, cqr.Attributes.RawMessageDelivery) + assert.Equal(t, FilterPolicy(nil), cqr.Attributes.FilterPolicy) +} + +func Test_DeleteMessageBatchRequest_SetAttributesFromForm_success(t *testing.T) { + form := url.Values{} + form.Add("DeleteMessageBatchRequestEntry.1.Id", "message-id-1") + form.Add("DeleteMessageBatchRequestEntry.1.ReceiptHandle", "receipt-handle-1") + form.Add("DeleteMessageBatchRequestEntry.2.Id", "message-id-2") + form.Add("DeleteMessageBatchRequestEntry.2.ReceiptHandle", "receipt-handle-2") + form.Add("DeleteMessageBatchRequestEntry.3.Id", "message-id-3") + form.Add("DeleteMessageBatchRequestEntry.3.ReceiptHandle", "receipt-handle-3") + + dmbr := &DeleteMessageBatchRequest{} + dmbr.SetAttributesFromForm(form) + + assert.Len(t, dmbr.Entries, 3) + assert.Equal(t, "message-id-1", dmbr.Entries[0].Id) + assert.Equal(t, "receipt-handle-1", dmbr.Entries[0].ReceiptHandle) + assert.Equal(t, "message-id-2", dmbr.Entries[1].Id) + assert.Equal(t, "receipt-handle-2", dmbr.Entries[1].ReceiptHandle) + assert.Equal(t, "message-id-3", dmbr.Entries[2].Id) + assert.Equal(t, "receipt-handle-3", dmbr.Entries[2].ReceiptHandle) +} + +func Test_DeleteMessageBatchRequest_SetAttributesFromForm_stops_at_non_sequential_keys(t *testing.T) { + form := url.Values{} + form.Add("DeleteMessageBatchRequestEntry.1.Id", "message-id-1") + form.Add("DeleteMessageBatchRequestEntry.1.ReceiptHandle", "receipt-handle-1") + form.Add("DeleteMessageBatchRequestEntry.4.Id", "message-id-2") + form.Add("DeleteMessageBatchRequestEntry.4.ReceiptHandle", "receipt-handle-2") + form.Add("DeleteMessageBatchRequestEntry.3.Id", "message-id-3") + form.Add("DeleteMessageBatchRequestEntry.3.ReceiptHandle", "receipt-handle-3") + + dmbr := &DeleteMessageBatchRequest{} + dmbr.SetAttributesFromForm(form) + + assert.Len(t, dmbr.Entries, 1) + assert.Equal(t, "message-id-1", dmbr.Entries[0].Id) + assert.Equal(t, "receipt-handle-1", dmbr.Entries[0].ReceiptHandle) +} + +func Test_DeleteMessageBatchRequest_SetAttributesFromForm_stops_at_invalid_keys(t *testing.T) { + form := url.Values{} + form.Add("DeleteMessageBatchRequestEntry.1.Id", "message-id-1") + form.Add("DeleteMessageBatchRequestEntry.1.ReceiptHandle", "receipt-handle-1") + form.Add("INVALID_DeleteMessageBatchRequestEntry.2.Id", "message-id-2") + form.Add("DeleteMessageBatchRequestEntry.2.ReceiptHandle", "receipt-handle-2") + form.Add("DeleteMessageBatchRequestEntry.3.Id", "message-id-3") + form.Add("DeleteMessageBatchRequestEntry.3.ReceiptHandle", "receipt-handle-3") + + dmbr := &DeleteMessageBatchRequest{} + dmbr.SetAttributesFromForm(form) + + assert.Len(t, dmbr.Entries, 1) + assert.Equal(t, "message-id-1", dmbr.Entries[0].Id) + assert.Equal(t, "receipt-handle-1", dmbr.Entries[0].ReceiptHandle) +} + +func TestPublishRequest_SetAttributesFromForm_success_concurrent(t *testing.T) { + form := url.Values{} + form.Add("MessageAttributes.entry.1.Name", "test1") + form.Add("MessageAttributes.entry.1.Value.DataType", "String") + form.Add("MessageAttributes.entry.1.Value.StringValue", "sample-string") + form.Add("MessageAttributes.entry.2.Name", "test2") + form.Add("MessageAttributes.entry.2.Value.DataType", "Binary") + form.Add("MessageAttributes.entry.2.Value.BinaryValue", "YmluYXJ5LXZhbHVl") + + // if the code is not thread-safe, repeated runs increase the chance of detecting a race. + for r := 0; r < 10; r++ { + var wg sync.WaitGroup + goroutineCount := 40 + // launch goroutines in parallel to simulate concurrent access. + for g := 0; g < goroutineCount; g++ { + wg.Add(1) + go func() { + defer wg.Done() + // introduce a random delay to encourage goroutine interleaving + time.Sleep(time.Duration(rand.Intn(5)) * time.Millisecond) + cqr := &PublishRequest{ + MessageAttributes: make(map[string]MessageAttribute), + } + + cqr.SetAttributesFromForm(form) + + // validate the expected DataType values + assert.Equal(t, "String", cqr.MessageAttributes["test1"].DataType) + assert.Equal(t, "Binary", cqr.MessageAttributes["test2"].DataType) + }() + } + wg.Wait() + } +} + +func TestParseMessageAttributes(t *testing.T) { + for _, tc := range []struct { + description string + values url.Values + keyPrefix string + want map[string]MessageAttribute + }{ + { + description: "empty", + values: url.Values{}, + keyPrefix: "foo", + want: nil, + }, + { + description: "simple", + values: url.Values{ + "MessageAttribute.1.Name": []string{"Attr1"}, + "MessageAttribute.1.Value.DataType": []string{"String"}, + "MessageAttribute.1.Value.StringValue": []string{"Value1"}, + "MessageAttribute.2.Name": []string{"Attr2"}, + "MessageAttribute.2.Value.DataType": []string{"Binary"}, + "MessageAttribute.2.Value.BinaryValue": []string{"VmFsdWUy"}, + }, + keyPrefix: "MessageAttribute", + want: map[string]MessageAttribute{ + "Attr1": { + DataType: "String", + StringValue: "Value1", + BinaryValue: "", + }, + "Attr2": { + DataType: "Binary", + BinaryValue: "VmFsdWUy", + }, + }, + }, + { + description: "attributes after empty name ignored", + values: url.Values{ + "MessageAttribute.1.Name": []string{""}, + "MessageAttribute.1.Value.DataType": []string{"String"}, + "MessageAttribute.1.Value.StringValue": []string{"Value4"}, + "MessageAttribute.2.Name": []string{"Attr2"}, + "MessageAttribute.2.Value.DataType": []string{"Binary"}, + "MessageAttribute.2.Value.BinaryValue": []string{"VmFsdWUy"}, + }, + keyPrefix: "MessageAttribute", + want: nil, + }, + { + description: "attributes after missing number ignored", + values: url.Values{ + // Note starting from 2 + "MessageAttribute.2.Name": []string{"Attr2"}, + "MessageAttribute.2.Value.DataType": []string{"Binary"}, + "MessageAttribute.2.Value.BinaryValue": []string{"VmFsdWUy"}, + }, + keyPrefix: "MessageAttribute", + want: nil, + }, + { + description: "empty DataType ignored", + values: url.Values{ + "MessageAttribute.1.Name": []string{"Attr4"}, + "MessageAttribute.1.Value.DataType": []string{""}, + "MessageAttribute.1.Value.StringValue": []string{"Value4"}, + }, + keyPrefix: "MessageAttribute", + want: nil, + }, + } { + t.Run(tc.description, func(t *testing.T) { + got := parseMessageAttributes(tc.values, tc.keyPrefix) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/shared-sqs/app/models/responses.go b/shared-sqs/app/models/responses.go new file mode 100644 index 0000000..8e438f2 --- /dev/null +++ b/shared-sqs/app/models/responses.go @@ -0,0 +1,340 @@ +package models + +import ( + "encoding/xml" +) + +type ResponseMetadata struct { + RequestId string `xml:"RequestId"` +} + +// NOTE: Every response in here MUST implement the `AbstractResponseBody` interface in order to be used +// in `encodeResponse` + +/*** Error Responses ***/ +type ErrorResult struct { + Type string `json:"Type,omitempty" xml:"Type,omitempty"` + Code string `json:"Code,omitempty" xml:"Code,omitempty"` + Message string `json:"Message,omitempty" xml:"Message,omitempty"` +} + +type ErrorResponse struct { + Result ErrorResult `json:"Error" xml:"Error"` + RequestId string `json:"RequestId" xml:"RequestId"` +} + +func (r ErrorResponse) GetResult() interface{} { + return r.Result +} + +func (r ErrorResponse) GetRequestId() string { + return r.RequestId +} + +/*** Receive Message Response */ +type ReceiveMessageResult struct { + Messages []*ResultMessage `json:"Messages" xml:"Message,omitempty"` +} + +type ReceiveMessageResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Result ReceiveMessageResult `json:"ReceiveMessageResult" xml:"ReceiveMessageResult"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r ReceiveMessageResponse) GetResult() interface{} { + return r.Result +} + +func (r ReceiveMessageResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +type ResultMessage struct { + MessageId string `json:"MessageId,omitempty" xml:"MessageId,omitempty"` + ReceiptHandle string `json:"ReceiptHandle,omitempty" xml:"ReceiptHandle,omitempty"` + MD5OfBody string `json:"MD5OfBody,omitempty" xml:"MD5OfBody,omitempty"` + Body string `json:"Body,omitempty" xml:"Body,omitempty"` + MD5OfMessageAttributes string `json:"MD5OfMessageAttributes,omitempty" xml:"MD5OfMessageAttributes,omitempty"` + MessageAttributes map[string]MessageAttribute `json:"MessageAttributes,omitempty" xml:"MessageAttribute,omitempty,attr"` + Attributes map[string]string `json:"Attributes,omitempty" xml:"Attribute,omitempty,attr"` +} + +// MarshalXML is a custom marshaler for the ResultMessage struct. We need it because we need to convert the +// maps into something that can be shown as XML. If we ever get rid of the XML response parsing this can go, +// and that would be glorious. +func (r *ResultMessage) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + type Attributes struct { + Name string `xml:"Name,omitempty"` + Value string `xml:"Value,omitempty"` + } + var attrs []Attributes + for key, value := range r.Attributes { + attribute := Attributes{ + Name: key, + Value: value, + } + attrs = append(attrs, attribute) + } + + type MessageAttributes struct { + Name string `xml:"Name,omitempty"` + Value MessageAttribute `xml:"Value,omitempty"` + } + var messageAttrs []MessageAttributes + for key, value := range r.MessageAttributes { + attribute := MessageAttributes{ + Name: key, + Value: value, + } + messageAttrs = append(messageAttrs, attribute) + } + e.EncodeToken(start) + + // Encode the fields + e.EncodeElement(r.MessageId, xml.StartElement{Name: xml.Name{Local: "MessageId"}}) + e.EncodeElement(r.ReceiptHandle, xml.StartElement{Name: xml.Name{Local: "ReceiptHandle"}}) + e.EncodeElement(r.MD5OfBody, xml.StartElement{Name: xml.Name{Local: "MD5OfBody"}}) + e.EncodeElement(r.Body, xml.StartElement{Name: xml.Name{Local: "Body"}}) + e.EncodeElement(attrs, xml.StartElement{Name: xml.Name{Local: "Attribute"}}) + e.EncodeElement(messageAttrs, xml.StartElement{Name: xml.Name{Local: "MessageAttribute"}}) + e.EncodeToken(xml.EndElement{Name: start.Name}) + return nil +} + +type ChangeMessageVisibilityResult struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r ChangeMessageVisibilityResult) GetResult() interface{} { + return nil +} + +func (r ChangeMessageVisibilityResult) GetRequestId() string { + return r.Metadata.RequestId +} + +/*** Create Queue Response */ +type CreateQueueResult struct { + QueueUrl string `json:"QueueUrl" xml:"QueueUrl"` +} + +type CreateQueueResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Result CreateQueueResult `json:"CreateQueueResult" xml:"CreateQueueResult"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r CreateQueueResponse) GetResult() interface{} { + return r.Result +} + +func (r CreateQueueResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +/*** List Queues Response */ +type ListQueuesResult struct { + // NOTE: the old XML sdks depend on QueueUrl, and the new JSON ones need QueueUrls + QueueUrls []string `json:"QueueUrls" xml:"QueueUrl"` +} + +type ListQueuesResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Result ListQueuesResult `json:"ListQueuesResult" xml:"ListQueuesResult"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r ListQueuesResponse) GetResult() interface{} { + return r.Result +} + +func (r ListQueuesResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +/*** Get Queue QueueAttributes ***/ +type Attribute struct { + Name string `json:"Name,omitempty" xml:"Name,omitempty"` + Value string `json:"Value,omitempty" xml:"Value,omitempty"` +} + +type GetQueueAttributesResult struct { + /* VisibilityTimeout, DelaySeconds, ReceiveMessageWaitTimeSeconds, ApproximateNumberOfMessages + ApproximateNumberOfMessagesNotVisible, CreatedTimestamp, LastModifiedTimestamp, QueueArn */ + Attrs []Attribute `json:"Attributes,omitempty" xml:"Attribute,omitempty"` +} + +type GetQueueAttributesResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Result GetQueueAttributesResult `json:"GetQueueAttributesResult" xml:"GetQueueAttributesResult"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r GetQueueAttributesResponse) GetResult() interface{} { + result := map[string]string{} + for _, attr := range r.Result.Attrs { + result[attr.Name] = attr.Value + } + return map[string]map[string]string{"Attributes": result} +} + +func (r GetQueueAttributesResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +/*** Send Message Response */ +type SendMessageResult struct { + MD5OfMessageAttributes string `json:"MD5OfMessageAttributes,omitempty" xml:"MD5OfMessageAttributes,omitempty"` + MD5OfMessageBody string `json:"MD5OfMessageBody" xml:"MD5OfMessageBody"` + MessageId string `json:"MessageId" xml:"MessageId"` + SequenceNumber string `json:"SequenceNumber,omitempty" xml:"SequenceNumber,omitempty"` +} + +type SendMessageResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Result SendMessageResult `json:"SendMessageResult" xml:"SendMessageResult"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r SendMessageResponse) GetResult() interface{} { + return r.Result +} + +func (r SendMessageResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +/*** Delete Message Response */ +type DeleteMessageResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r DeleteMessageResponse) GetResult() interface{} { + return nil +} + +func (r DeleteMessageResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +/*** Get Queue Url Response */ +type GetQueueUrlResult struct { + QueueUrl string `json:"QueueUrl,omitempty" xml:"QueueUrl,omitempty"` +} + +type GetQueueUrlResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Result GetQueueUrlResult `json:"GetQueueUrlResult" xml:"GetQueueUrlResult"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r GetQueueUrlResponse) GetResult() interface{} { + return r.Result +} + +func (r GetQueueUrlResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +type SendMessageBatchResultEntry struct { + Id string `json:"Id" xml:"Id"` + MessageId string `json:"MessageId" xml:"MessageId"` + MD5OfMessageBody string `json:"MD5OfMessageBody,omitempty" xml:"MD5OfMessageBody,omitempty"` + MD5OfMessageAttributes string `json:"MD5OfMessageAttributes,omitempty" xml:"MD5OfMessageAttributes,omitempty"` + SequenceNumber string `json:"SequenceNumber" xml:"SequenceNumber"` +} + +/*** Send Message Batch Response */ +type SendMessageBatchResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Result SendMessageBatchResult `json:"SendMessageBatchResult" xml:"SendMessageBatchResult"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +type SendMessageBatchResult struct { + Entry []SendMessageBatchResultEntry `json:"SendMessageBatchResultEntry" xml:"SendMessageBatchResultEntry"` + Error []BatchResultErrorEntry `json:"BatchResultErrorEntry,omitempty" xml:"BatchResultErrorEntry,omitempty"` +} + +func (r SendMessageBatchResponse) GetResult() interface{} { + return r.Result +} + +func (r SendMessageBatchResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +type BatchResultErrorEntry struct { + Code string `json:"Code" xml:"Code"` + Id string `json:"Id" xml:"Id"` + Message string `json:"Message,omitempty" xml:"Message,omitempty"` + SenderFault bool `json:"SenderFault" xml:"SenderFault"` +} + +type SetQueueAttributesResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r SetQueueAttributesResponse) GetResult() interface{} { + return nil +} + +func (r SetQueueAttributesResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +/*** Purge Queue Response */ +type PurgeQueueResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r PurgeQueueResponse) GetResult() interface{} { + return nil +} + +func (r PurgeQueueResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +/*** Delete Queue Response */ +type DeleteQueueResponse struct { + Xmlns string `json:"Xmlns" xml:"xmlns,attr"` + Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r DeleteQueueResponse) GetResult() interface{} { + return nil +} + +func (r DeleteQueueResponse) GetRequestId() string { + return r.Metadata.RequestId +} + +/*** Delete Message Batch Response ***/ +type DeleteMessageBatchResultEntry struct { +Id string `json:"Id" xml:"Id"` +} + +type DeleteMessageBatchResult struct { +Successful []DeleteMessageBatchResultEntry `json:"DeleteMessageBatchResultEntry" xml:"DeleteMessageBatchResultEntry"` +Failed []BatchResultErrorEntry `json:"BatchResultErrorEntry,omitempty" xml:"BatchResultErrorEntry,omitempty"` +} + +type DeleteMessageBatchResponse struct { +Xmlns string `json:"Xmlns" xml:"xmlns,attr"` +Result DeleteMessageBatchResult `json:"DeleteMessageBatchResult" xml:"DeleteMessageBatchResult"` +Metadata ResponseMetadata `json:"ResponseMetadata" xml:"ResponseMetadata"` +} + +func (r DeleteMessageBatchResponse) GetResult() interface{} { +return r.Result +} + +func (r DeleteMessageBatchResponse) GetRequestId() string { +return r.Metadata.RequestId +} diff --git a/shared-sqs/app/models/responses_test.go b/shared-sqs/app/models/responses_test.go new file mode 100644 index 0000000..92b7541 --- /dev/null +++ b/shared-sqs/app/models/responses_test.go @@ -0,0 +1,110 @@ +package models + +import ( + "encoding/xml" + "testing" + + "github.com/stretchr/testify/assert" +) + +// NOTE: For now, we're only going to test those methods that do something other than just return a field + +func TestGetQueueAttributesResponse_GetResult(t *testing.T) { + gqa := GetQueueAttributesResponse{ + Result: GetQueueAttributesResult{Attrs: []Attribute{ + {Name: "attribute-name1", Value: "attribute-value1"}, + {Name: "attribute-name2", Value: "attribute-value2"}, + }}, + } + + expectedAttributes := map[string]map[string]string{ + "Attributes": { + "attribute-name1": "attribute-value1", + "attribute-name2": "attribute-value2", + }, + } + result := gqa.GetResult() + + assert.Equal(t, expectedAttributes, result) +} + +func Test_ResultMessage_MarshalXML_success_with_attributes(t *testing.T) { + input := &ResultMessage{ + MessageId: "message-id", + ReceiptHandle: "receipt-handle", + MD5OfBody: "body-md5", + Body: "message-body", + MD5OfMessageAttributes: "message-attrs-md5", + MessageAttributes: map[string]MessageAttribute{ + "attr1": { + DataType: "String", + StringValue: "string-value", + }, + "attr2": { + DataType: "Binary", + BinaryValue: "binary-value", + }, + "attr3": { + DataType: "Number", + StringValue: "number-value", + }, + }, + Attributes: map[string]string{ + "ApproximateFirstReceiveTimestamp": "1", + "SenderId": "2", + "ApproximateReceiveCount": "3", + "SentTimestamp": "4", + }, + } + result, err := xml.Marshal(input) + + assert.Nil(t, err) + + resultString := string(result) + + // We have to assert piecemeal like this, the maps go into their lists unordered, which will randomly break this. + entry := "message-idreceipt-handlebody-md5message-body" + assert.Contains(t, resultString, entry) + + entry = "ApproximateFirstReceiveTimestamp1" + assert.Contains(t, resultString, entry) + + entry = "SenderId2" + assert.Contains(t, resultString, entry) + + entry = "ApproximateReceiveCount3" + assert.Contains(t, resultString, entry) + + entry = "SentTimestamp4" + assert.Contains(t, resultString, entry) + + entry = "attr1Stringstring-value" + assert.Contains(t, resultString, entry) + + entry = "attr2binary-valueBinary" + assert.Contains(t, resultString, entry) + + entry = "attr3Numbernumber-value" + assert.Contains(t, resultString, entry) + + entry = "" + assert.Contains(t, resultString, entry) +} + +func Test_ResultMessage_MarshalXML_success_no_attributes(t *testing.T) { + input := &ResultMessage{ + MessageId: "message-id", + ReceiptHandle: "receipt-handle", + MD5OfBody: "body-md5", + Body: "message-body", + MD5OfMessageAttributes: "message-attrs-md5", + } + expectedOutput := "message-idreceipt-handlebody-md5message-body" + + result, err := xml.Marshal(input) + + assert.Nil(t, err) + + resultString := string(result) + assert.Equal(t, resultString, expectedOutput) +} diff --git a/shared-sqs/app/router/router.go b/shared-sqs/app/router/router.go new file mode 100644 index 0000000..39a611c --- /dev/null +++ b/shared-sqs/app/router/router.go @@ -0,0 +1,125 @@ +package router + +import ( +"encoding/json" +"encoding/xml" +"fmt" +"io" +"net/http" +"strings" + +"shared-sqs/app/interfaces" + +log "github.com/sirupsen/logrus" + +sqs "shared-sqs/app/gosqs" +"github.com/gorilla/mux" +) + +// New returns a new router +func New() http.Handler { +r := mux.NewRouter() + +r.HandleFunc("/", actionHandler).Methods("GET", "POST") +r.HandleFunc("/health", health).Methods("GET") +r.HandleFunc("/{account}", actionHandler).Methods("GET", "POST") +r.HandleFunc("/queue/{queueName}", actionHandler).Methods("GET", "POST") +r.HandleFunc("/{account}/{queueName}", actionHandler).Methods("GET", "POST") + +return r +} + +func encodeResponse(w http.ResponseWriter, req *http.Request, statusCode int, body interfaces.AbstractResponseBody) { +protocol := resolveProtocol(req) +switch protocol { +case AwsJsonProtocol: +w.Header().Set("x-amzn-RequestId", body.GetRequestId()) +w.Header().Set("Content-Type", "application/x-amz-json-1.0") +w.WriteHeader(statusCode) +if body.GetResult() == nil { +return +} +err := json.NewEncoder(w).Encode(body.GetResult()) +if err != nil { +log.Errorf("Response Encoding Error: %v\nResponse: %+v", err, body) +http.Error(w, "General Error", http.StatusInternalServerError) +} +case AwsQueryProtocol: +w.Header().Set("Content-Type", "application/xml") +w.WriteHeader(statusCode) +result, err := xml.Marshal(body) +if err != nil { +log.Errorf("Response Encoding Error: %v\nResponse: %+v", err, body) +http.Error(w, "General Error", http.StatusInternalServerError) +} +_, _ = w.Write(result) +} +} + +// routingTableV1 содержит только SQS actions — SNS удалён +var routingTableV1 = map[string]func(r *http.Request) (int, interfaces.AbstractResponseBody){ +"CreateQueue": sqs.CreateQueueV1, +"ListQueues": sqs.ListQueuesV1, +"GetQueueAttributes": sqs.GetQueueAttributesV1, +"SetQueueAttributes": sqs.SetQueueAttributesV1, +"SendMessage": sqs.SendMessageV1, +"ReceiveMessage": sqs.ReceiveMessageV1, +"ChangeMessageVisibility": sqs.ChangeMessageVisibilityV1, +"DeleteMessage": sqs.DeleteMessageV1, +"GetQueueUrl": sqs.GetQueueUrlV1, +"PurgeQueue": sqs.PurgeQueueV1, +"DeleteQueue": sqs.DeleteQueueV1, +"SendMessageBatch": sqs.SendMessageBatchV1, +"DeleteMessageBatch": sqs.DeleteMessageBatchV1, +} + +func health(w http.ResponseWriter, req *http.Request) { +w.WriteHeader(200) +fmt.Fprint(w, "OK") +} + +func actionHandler(w http.ResponseWriter, req *http.Request) { +action := extractAction(req) +log.WithFields( +log.Fields{ +"action": action, +"url": req.URL, +}).Debug("Handling URL request") +jsonFn, ok := routingTableV1[action] +if ok { +statusCode, responseBody := jsonFn(req) +encodeResponse(w, req, statusCode, responseBody) +return +} +log.Warnf("Bad Request - Action: %s", action) +w.WriteHeader(http.StatusBadRequest) +io.WriteString(w, "Bad Request") +} + +type AwsProtocol int + +const ( +AwsJsonProtocol AwsProtocol = iota +AwsQueryProtocol AwsProtocol = iota +) + +// extractAction — извлекает Action из запроса (Query Protocol или JSON Protocol) +func extractAction(req *http.Request) string { +protocol := resolveProtocol(req) +switch protocol { +case AwsJsonProtocol: +action := req.Header.Get("X-Amz-Target") +return strings.Split(action, ".")[1] +case AwsQueryProtocol: +return req.FormValue("Action") +} +return "" +} + +// resolveProtocol — определяет протокол по Content-Type +func resolveProtocol(req *http.Request) AwsProtocol { +if req.Header.Get("Content-Type") == "application/x-amz-json-1.0" { +return AwsJsonProtocol +} +return AwsQueryProtocol +} diff --git a/shared-sqs/app/router/router_test.go b/shared-sqs/app/router/router_test.go new file mode 100644 index 0000000..dd06773 --- /dev/null +++ b/shared-sqs/app/router/router_test.go @@ -0,0 +1,251 @@ +package router + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + af "shared-sqs/app/fixtures" + + "shared-sqs/app/mocks" + + "shared-sqs/app/interfaces" + + sqs "shared-sqs/app/gosqs" + + "github.com/stretchr/testify/assert" + + "shared-sqs/app/test" +) + +func TestIndexServerhandler_POST_BadRequest(t *testing.T) { + // Create a request to pass to our handler. We don't have any query parameters for now, so we'll + // pass 'nil' as the third parameter. + req, err := http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form := url.Values{} + form.Add("Action", "BadRequest") + req.PostForm = form + + // We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response. + rr := httptest.NewRecorder() + + // Our handlers satisfy http.Handler, so we can call their ServeHTTP method + // directly and pass in our Request and ResponseRecorder. + New().ServeHTTP(rr, req) + + // Check the status code is what we expect. + if status := rr.Code; status != http.StatusBadRequest { + t.Errorf("handler returned wrong status code: got %v want %v", + status, http.StatusOK) + } +} + +func TestIndexServerhandler_POST_GoodRequest(t *testing.T) { + // Create a request to pass to our handler. We don't have any query parameters for now, so we'll + // pass 'nil' as the third parameter. + req, err := http.NewRequest("POST", "/", nil) + if err != nil { + t.Fatal(err) + } + + form := url.Values{} + form.Add("Action", "ListTopics") + req.PostForm = form + + // We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response. + rr := httptest.NewRecorder() + + // Our handlers satisfy http.Handler, so we can call their ServeHTTP method + // directly and pass in our Request and ResponseRecorder. + New().ServeHTTP(rr, req) + + // Check the status code is what we expect. + if status := rr.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code: got %v want %v", + status, http.StatusOK) + } +} + +func TestIndexServerhandler_POST_GoodRequest_With_URL(t *testing.T) { + req, err := http.NewRequest("POST", "/100010001000/local-queue1", nil) + if err != nil { + t.Fatal(err) + } + + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "local-queue1") + req.PostForm = form + rr := httptest.NewRecorder() + New().ServeHTTP(rr, req) + + form = url.Values{} + form.Add("Action", "GetQueueAttributes") + form.Add("QueueUrl", fmt.Sprintf("%s/local-queue1", af.BASE_URL)) + req.PostForm = form + + // We create a ResponseRecorder (which satisfies http.ResponseWriter) to record the response. + rr = httptest.NewRecorder() + + // Our handlers satisfy http.Handler, so we can call their ServeHTTP method + // directly and pass in our Request and ResponseRecorder. + New().ServeHTTP(rr, req) + + assert.Equal(t, http.StatusOK, rr.Code) +} + +func TestIndexServerhandler_POST_GoodRequest_With_URL_And_Aws_Json_Protocol(t *testing.T) { + json, _ := json.Marshal(map[string]string{ + "QueueName": "local-queue1", + }) + req, err := http.NewRequest("POST", "/100010001000/local-queue1", bytes.NewBuffer(json)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Amz-Target", "AmazonSQS.CreateQueue") + req.Header.Set("Content-Type", "application/x-amz-json-1.0") + + rr := httptest.NewRecorder() + + New().ServeHTTP(rr, req) + + // Check the status code is what we expect. + if status := rr.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code: got %v want %v", + status, http.StatusOK) + } +} + +func TestIndexServerhandler_GET_GoodRequest_Pem_cert(t *testing.T) { + + req, err := http.NewRequest("GET", "/SimpleNotificationService/100010001000.pem", nil) + if err != nil { + t.Fatal(err) + } + + rr := httptest.NewRecorder() + New().ServeHTTP(rr, req) + + if status := rr.Code; status != http.StatusOK { + t.Errorf("handler returned wrong status code: got %v want %v", + status, http.StatusOK) + } +} + +func TestEncodeResponse_success_xml(t *testing.T) { + w, r := test.GenerateRequestInfo("POST", "/url", nil, false) + + encodeResponse(w, r, http.StatusOK, mocks.BaseResponse{Message: "test"}) + + assert.Equal(t, http.StatusOK, w.Code) + + tmp := mocks.BaseResponse{} + xml.Unmarshal(w.Body.Bytes(), &tmp) + assert.Equal(t, mocks.BaseResponse{Message: "test"}, tmp) +} + +func TestEncodeResponse_success_skips_nil_body_xml(t *testing.T) { + w, r := test.GenerateRequestInfo("POST", "/url", nil, false) + + encodeResponse(w, r, http.StatusOK, nil) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, &bytes.Buffer{}, w.Body) +} + +func TestEncodeResponse_success_json(t *testing.T) { + w, r := test.GenerateRequestInfo("POST", "/url", nil, true) + + encodeResponse(w, r, http.StatusOK, mocks.BaseResponse{Message: "test"}) + + assert.Equal(t, http.StatusOK, w.Code) + + tmp := mocks.BaseResponse{} + json.Unmarshal(w.Body.Bytes(), &tmp) + assert.Equal(t, mocks.BaseResponse{Message: "test"}, tmp) +} + +func TestEncodeResponse_success_skips_malformed_body_json(t *testing.T) { + mock := mocks.BaseResponse{ + Message: "test", + } + mock.MockGetResult = func() interface{} { + return make(chan int) + } + w, r := test.GenerateRequestInfo("POST", "/url", nil, true) + + encodeResponse(w, r, http.StatusOK, mock) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "General Error", strings.TrimSpace(string(w.Body.Bytes()))) +} + +func TestActionHandler_v1_json(t *testing.T) { + defer func() { + routingTableV1 = map[string]func(r *http.Request) (int, interfaces.AbstractResponseBody){ + "CreateQueue": sqs.CreateQueueV1, + } + }() + + mockCalled := false + mockFunction := func(req *http.Request) (int, interfaces.AbstractResponseBody) { + mockCalled = true + return http.StatusOK, mocks.BaseResponse{Message: "response-body"} + } + routingTableV1 = map[string]func(r *http.Request) (int, interfaces.AbstractResponseBody){ + "CreateQueue": mockFunction, + } + + w, r := test.GenerateRequestInfo("POST", "/url", nil, true) + r.Header.Set("X-Amz-Target", "QueueService.CreateQueue") + + actionHandler(w, r) + + assert.True(t, mockCalled) + assert.Equal(t, http.StatusOK, w.Code) + + tmp := mocks.BaseResponse{} + json.Unmarshal(w.Body.Bytes(), &tmp) + assert.Equal(t, mocks.BaseResponse{Message: "response-body"}, tmp) +} + +func TestActionHandler_v1_xml(t *testing.T) { + defer func() { + routingTableV1 = map[string]func(r *http.Request) (int, interfaces.AbstractResponseBody){ + "CreateQueue": sqs.CreateQueueV1, + } + }() + + mockCalled := false + mockFunction := func(req *http.Request) (int, interfaces.AbstractResponseBody) { + mockCalled = true + return http.StatusOK, mocks.BaseResponse{Message: "response-body"} + } + routingTableV1 = map[string]func(r *http.Request) (int, interfaces.AbstractResponseBody){ + "CreateQueue": mockFunction, + } + + w, r := test.GenerateRequestInfo("POST", "/url", nil, false) + form := url.Values{} + form.Add("Action", "CreateQueue") + r.PostForm = form + + actionHandler(w, r) + + assert.True(t, mockCalled) + assert.Equal(t, http.StatusOK, w.Code) + + tmp := mocks.BaseResponse{} + xml.Unmarshal(w.Body.Bytes(), &tmp) + assert.Equal(t, mocks.BaseResponse{Message: "response-body"}, tmp) +} diff --git a/shared-sqs/app/utils/utils.go b/shared-sqs/app/utils/utils.go new file mode 100644 index 0000000..43bf7d9 --- /dev/null +++ b/shared-sqs/app/utils/utils.go @@ -0,0 +1,148 @@ +package utils + +import ( + "crypto/md5" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "hash" + "io" + "net/http" + "net/url" + "sort" + "strings" + + "shared-sqs/app/models" + + "shared-sqs/app/interfaces" + + log "github.com/sirupsen/logrus" + + "github.com/gorilla/schema" +) + +var XmlDecoder *schema.Decoder +var REQUEST_TRANSFORMER = TransformRequest + +func init() { + XmlDecoder = schema.NewDecoder() + XmlDecoder.IgnoreUnknownKeys(true) +} + +func TransformRequest(resultingStruct interfaces.AbstractRequestBody, req *http.Request, emptyRequestValid bool) (success bool) { + switch req.Header.Get("Content-Type") { + case "application/x-amz-json-1.0": + //Read body data to parse json + decoder := json.NewDecoder(req.Body) + err := decoder.Decode(resultingStruct) + if err != nil { + if emptyRequestValid && err == io.EOF { + return true + } + log.Debugf("TransformRequest Failure - %s", err.Error()) + return false + } + default: + err := req.ParseForm() + if err != nil { + log.Debugf("TransformRequest Failure - %s", err.Error()) + return false + } + err = XmlDecoder.Decode(resultingStruct, req.PostForm) + if err != nil { + log.Debugf("TransformRequest Failure - %s", err.Error()) + return false + } + resultingStruct.SetAttributesFromForm(req.PostForm) + } + + return true +} + +func ExtractQueueAttributes(u url.Values) map[string]string { + attr := map[string]string{} + for i := 1; true; i++ { + nameKey := fmt.Sprintf("Attribute.%d.Name", i) + attrName := u.Get(nameKey) + if attrName == "" { + break + } + + valueKey := fmt.Sprintf("Attribute.%d.Value", i) + attrValue := u.Get(valueKey) + if attrValue != "" { + attr[attrName] = attrValue + } + } + return attr +} + +func CreateErrorResponseV1(errKey string, isSqs bool) (int, interfaces.AbstractResponseBody) { + var err interfaces.AbstractErrorResponse + if isSqs { + err = models.SqsErrors[errKey] + } else { + err = models.SnsErrors[errKey] + } + + respStruct := models.ErrorResponse{ + Result: err.Response(), + RequestId: "00000000-0000-0000-0000-000000000000", + } + return err.StatusCode(), respStruct +} + +func GetMD5Hash(text string) string { + hasher := md5.New() + hasher.Write([]byte(text)) + return hex.EncodeToString(hasher.Sum(nil)) +} + +func HashAttributes(attributes map[string]models.MessageAttribute) string { + hasher := md5.New() + + keys := sortedKeys(attributes) + for _, key := range keys { + attributeValue := attributes[key] + + addStringToHash(hasher, key) + addStringToHash(hasher, attributeValue.DataType) + if attributeValue.DataType == "String" { + hasher.Write([]byte{1}) + addStringToHash(hasher, attributeValue.StringValue) + } else if attributeValue.DataType == "Binary" { + hasher.Write([]byte{2}) + bytes, _ := base64.StdEncoding.DecodeString(attributeValue.BinaryValue) + addBytesToHash(hasher, []byte(bytes)) + } + } + + return hex.EncodeToString(hasher.Sum(nil)) +} + +func sortedKeys(attributes map[string]models.MessageAttribute) []string { + var keys []string + for key := range attributes { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func addStringToHash(hasher hash.Hash, str string) { + bytes := []byte(str) + addBytesToHash(hasher, bytes) +} + +func addBytesToHash(hasher hash.Hash, arr []byte) { + bs := make([]byte, 4) + binary.BigEndian.PutUint32(bs, uint32(len(arr))) + hasher.Write(bs) + hasher.Write(arr) +} + +func HasFIFOQueueName(queueName string) bool { + return strings.HasSuffix(queueName, ".fifo") +} diff --git a/shared-sqs/app/utils/utils_test.go b/shared-sqs/app/utils/utils_test.go new file mode 100644 index 0000000..72e978d --- /dev/null +++ b/shared-sqs/app/utils/utils_test.go @@ -0,0 +1,139 @@ +package utils + +import ( + "net/url" + "testing" + + "shared-sqs/app/models" + + "shared-sqs/app/test" + + "shared-sqs/app/fixtures" + "shared-sqs/app/mocks" + + "github.com/stretchr/testify/assert" +) + +func TestTransformRequest_success_json(t *testing.T) { + _, r := test.GenerateRequestInfo("POST", "url", fixtures.JSONRequestBody, true) + + mock := &mocks.MockRequestBody{} + + ok := TransformRequest(mock, r, false) + + assert.True(t, ok) + assert.Equal(t, "mock-value", mock.RequestFieldStr) + assert.False(t, mock.SetAttributesFromFormCalled) +} + +func TestTransformRequest_success_json_empty_request_accepted(t *testing.T) { + _, r := test.GenerateRequestInfo("POST", "url", nil, true) + + mock := &mocks.MockRequestBody{} + + ok := TransformRequest(mock, r, true) + + assert.True(t, ok) + //assert.Equal(t, "mock-value", mock.RequestFieldStr) + assert.False(t, mock.SetAttributesFromFormCalled) +} + +func TestTransformRequest_success_xml(t *testing.T) { + _, r := test.GenerateRequestInfo("POST", "url", nil, false) + form := url.Values{} + form.Add("Action", "CreateQueue") + form.Add("QueueName", "UnitTestQueue1") + form.Add("Attribute.1.Name", "VisibilityTimeout") + form.Add("Attribute.1.Value", "60") + form.Add("Attribute.2.Name", "MaximumMessageSize") + form.Add("Attribute.2.Value", "2048") + r.PostForm = form + + mock := &mocks.MockRequestBody{} + + ok := TransformRequest(mock, r, false) + + assert.True(t, ok) + assert.True(t, mock.SetAttributesFromFormCalled) + assert.Equal(t, []interface{}{form}, mock.SetAttributesFromFormCalledWith) +} + +func TestTransformRequest_error_invalid_request_body_json(t *testing.T) { + _, r := test.GenerateRequestInfo("POST", "url", "\"I-am-garbage", true) + + mock := &mocks.MockRequestBody{} + + ok := TransformRequest(mock, r, false) + + assert.False(t, ok) + assert.Equal(t, "", mock.RequestFieldStr) + assert.False(t, mock.SetAttributesFromFormCalled) +} + +func TestTransformRequest_error_failure_to_parse_form_xml(t *testing.T) { + _, r := test.GenerateRequestInfo("POST", "url", nil, false) + + mock := &mocks.MockRequestBody{} + + ok := TransformRequest(mock, r, false) + + assert.False(t, ok) + assert.False(t, mock.SetAttributesFromFormCalled) +} + +func TestTransformRequest_error_invalid_request_body_xml(t *testing.T) { + _, r := test.GenerateRequestInfo("POST", "url", nil, false) + + form := url.Values{} + form.Add("intField", "\"I-am-garbage") + r.PostForm = form + + mock := &mocks.MockRequestBody{} + + ok := TransformRequest(mock, r, false) + + assert.False(t, ok) + assert.False(t, mock.SetAttributesFromFormCalled) +} + +func TestExtractQueueAttributes_success(t *testing.T) { + u := url.Values{} + u.Add("Attribute.1.Name", "DelaySeconds") + u.Add("Attribute.1.Value", "20") + u.Add("Attribute.2.Name", "VisibilityTimeout") + u.Add("Attribute.2.Value", "30") + u.Add("Attribute.3.Name", "Policy") + + attr := ExtractQueueAttributes(u) + expected := map[string]string{ + "DelaySeconds": "20", + "VisibilityTimeout": "30", + } + + assert.Equal(t, expected, attr) +} + +func TestGetMD5Hash(t *testing.T) { + hash1 := GetMD5Hash("This is a test") + hash2 := GetMD5Hash("This is a test") + if hash1 != hash2 { + t.Errorf("hashs and hash2 should be the same, but were not") + } + + hash1 = GetMD5Hash("This is a test") + hash2 = GetMD5Hash("This is a tfst") + if hash1 == hash2 { + t.Errorf("hashs and hash2 are the same, but should not be") + } +} + +func TestSortedKeys(t *testing.T) { + attributes := map[string]models.MessageAttribute{ + "b": {}, + "a": {}, + } + + keys := sortedKeys(attributes) + assert.Equal(t, "a", keys[0]) + assert.Equal(t, "b", keys[1]) +} diff --git a/shared-sqs/go.mod b/shared-sqs/go.mod new file mode 100644 index 0000000..1ae1e16 --- /dev/null +++ b/shared-sqs/go.mod @@ -0,0 +1,30 @@ +module shared-sqs + +go 1.22 + +require ( + github.com/ghodss/yaml v1.0.0 + github.com/google/uuid v1.6.0 + github.com/gorilla/mux v1.8.0 + github.com/gorilla/schema v1.4.1 + github.com/mitchellh/copystructure v1.2.0 + github.com/sirupsen/logrus v1.9.0 + github.com/stretchr/testify v1.7.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/pretty v0.1.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/sys v0.13.0 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0 // indirect +) + +retract ( + v1.1.2 + v1.1.1 + v1.1.0 +) diff --git a/shared-sqs/go.sum b/shared-sqs/go.sum new file mode 100644 index 0000000..f2c5616 --- /dev/null +++ b/shared-sqs/go.sum @@ -0,0 +1,38 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E= +github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=