#!/bin/bash # test_full_suite.sh — Полный тест-сьют SQS Operator v0.1.4 # Создан: 2026-04-07 # Длительность: ~30 минут # 6 фаз: базовые, ошибочные, продвинутые, multi-tenant, self-healing, стресс-марафон set -o pipefail # === КОНФИГ === AK=$(kubectl -n sless-fn-test001 get secret sqs-creds-test001 -o jsonpath='{.data.accessKey}' | base64 -d) SK=$(kubectl -n sless-fn-test001 get secret sqs-creds-test001 -o jsonpath='{.data.secretKey}' | base64 -d) EP="https://sqs.kube5s.ru/sqs/test001" # Счётчики PASS=0; FAIL=0; WARN=0 FAIL_LIST="" WARN_LIST="" # === УТИЛИТЫ === ok() { echo " ✅ PASS[$1]"; PASS=$((PASS+1)); } fail() { echo " ❌ FAIL[$1]: $2"; FAIL=$((FAIL+1)); FAIL_LIST="${FAIL_LIST}\n - $1: $2"; } warn() { echo " ⚠️ WARN[$1]: $2"; WARN=$((WARN+1)); WARN_LIST="${WARN_LIST}\n - $1: $2"; } hdr() { echo ""; echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; echo " $1"; echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; } # GET-запрос к SQS sqs() { curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" "$EP/?$1" } # POST-запрос (нужен для ReceiptHandle с #) sqspost() { curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" \ -X POST -H "Content-Type: application/x-www-form-urlencoded" \ --data "$1" "$EP/" } # Encode # → %23 в ReceiptHandle encode_receipt() { echo "$1" | sed 's/#/%23/g'; } # Ждать Phase=Ready у QueueService wait_qs_ready() { local name="$1" ns="${2:-sqs-operator-system}" max="${3:-60}" for i in $(seq 1 $max); do local p=$(kubectl get queueservice "$name" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null) [ "$p" = "Ready" ] && { echo "Ready"; return 0; } sleep 2 done echo "Timeout" return 1 } echo "==========================================" echo " SQS Operator Full Test Suite v0.1.4" echo " $(date '+%Y-%m-%d %H:%M:%S')" echo " tenant: test001 ep: $EP" echo "==========================================" echo " AK: $AK" [ -z "$AK" ] && { echo "FATAL: нет credentials, выход."; exit 1; } # ───────────────────────────────────────────── hdr "PHASE 1 — Базовые операции (T01–T11)" # ───────────────────────────────────────────── # T01: ListQueues echo "--- T01 ListQueues" R=$(sqs "Action=ListQueues&Version=2012-11-05") echo "$R" | grep -q "ListQueuesResponse" && ok T01 || fail T01 "нет ListQueuesResponse: $R" # T02: CreateQueue echo "--- T02 CreateQueue" R=$(sqs "Action=CreateQueue&QueueName=t-basic&Version=2012-11-05") BASIC_URL=$(echo "$R" | grep -oP "(?<=)[^<]+") [ -n "$BASIC_URL" ] && ok "T02 ($BASIC_URL)" || fail T02 "нет QueueUrl: $R" # T03: CreateQueue duplicate (idempotency) echo "--- T03 CreateQueue idempotent" R=$(sqs "Action=CreateQueue&QueueName=t-basic&Version=2012-11-05") URL2=$(echo "$R" | grep -oP "(?<=)[^<]+") [ "$URL2" = "$BASIC_URL" ] && ok T03 || fail T03 "URL изменился: $BASIC_URL vs $URL2" # T04: GetQueueUrl echo "--- T04 GetQueueUrl" R=$(sqs "Action=GetQueueUrl&QueueName=t-basic&Version=2012-11-05") echo "$R" | grep -q "QueueUrl" && ok T04 || fail T04 "$R" # T05: SendMessage echo "--- T05 SendMessage" R=$(sqs "Action=SendMessage&QueueUrl=${BASIC_URL}&MessageBody=hello-world&Version=2012-11-05") MSG_ID=$(echo "$R" | grep -oP "(?<=)[^<]+") [ -n "$MSG_ID" ] && ok "T05 (id=$MSG_ID)" || fail T05 "$R" # T06: ReceiveMessage echo "--- T06 ReceiveMessage" R=$(sqs "Action=ReceiveMessage&QueueUrl=${BASIC_URL}&MaxNumberOfMessages=1&Version=2012-11-05") BODY=$(echo "$R" | grep -oP "(?<=)[^<]+" | head -1) RECEIPT06=$(echo "$R" | grep -oP "(?<=)[^<]+" | head -1) [ "$BODY" = "hello-world" ] && ok "T06 (body=$BODY)" || fail T06 "body=$BODY, R=$R" # T07: DeleteMessage via POST (# в ReceiptHandle) echo "--- T07 DeleteMessage" ENC=$(encode_receipt "$RECEIPT06") R=$(sqspost "Action=DeleteMessage&QueueUrl=${BASIC_URL}&ReceiptHandle=${ENC}&Version=2012-11-05") echo "$R" | grep -q "DeleteMessageResponse\|ResponseMetadata" && ok T07 || fail T07 "$R" # T08: ReceiveMessage after delete (пусто) echo "--- T08 ReceiveMessage empty" sleep 1 R=$(sqs "Action=ReceiveMessage&QueueUrl=${BASIC_URL}&MaxNumberOfMessages=1&WaitTimeSeconds=0&Version=2012-11-05") echo "$R" | grep -q "" && fail T08 "сообщение ещё есть после delete" || ok T08 # T09: GetQueueAttributes (All) echo "--- T09 GetQueueAttributes" R=$(sqs "Action=GetQueueAttributes&QueueUrl=${BASIC_URL}&AttributeName.1=All&Version=2012-11-05") echo "$R" | grep -q "VisibilityTimeout" && ok T09 || fail T09 "$R" # T10: SetQueueAttributes echo "--- T10 SetQueueAttributes" R=$(sqs "Action=SetQueueAttributes&QueueUrl=${BASIC_URL}&Attribute.1.Name=VisibilityTimeout&Attribute.1.Value=10&Version=2012-11-05") echo "$R" | grep -q "ResponseMetadata" && ok T10 || fail T10 "$R" # T11: DeleteQueue echo "--- T11 DeleteQueue" R=$(sqs "Action=DeleteQueue&QueueUrl=${BASIC_URL}&Version=2012-11-05") echo "$R" | grep -q "ResponseMetadata" && ok T11 || fail T11 "$R" # ───────────────────────────────────────────── hdr "PHASE 2 — Ошибочные параметры (E01–E09)" # ───────────────────────────────────────────── # E01: InvalidQueueName (спецсимволы) echo "--- E01 CreateQueue invalid name (spa ces)" R=$(sqs "Action=CreateQueue&QueueName=bad%20name%21&Version=2012-11-05") echo "$R" | grep -qi "error\|invalid" && ok E01 || warn E01 "elasticmq принял: $R" # E02: VisibilityTimeout > 43200 (12 часов — максимум AWS) echo "--- E02 VisibilityTimeout > 43200" sqs "Action=CreateQueue&QueueName=t-err02&Version=2012-11-05" > /dev/null ERR2_URL=$(sqs "Action=GetQueueUrl&QueueName=t-err02&Version=2012-11-05" | grep -oP "(?<=)[^<]+") R=$(sqs "Action=SetQueueAttributes&QueueUrl=${ERR2_URL}&Attribute.1.Name=VisibilityTimeout&Attribute.1.Value=99999&Version=2012-11-05") echo "$R" | grep -qi "error\|invalid" && ok E02 || warn E02 "accepted invalidTimeout: $R" # E03: ReceiveMessage от несуществующей очереди echo "--- E03 ReceiveMessage non-existent queue" R=$(sqs "Action=ReceiveMessage&QueueUrl=${EP}/test001/queue-xyz-does-not-exist&Version=2012-11-05") echo "$R" | grep -qi "error\|NonExistent\|not exist" && ok E03 || fail E03 "нет ошибки: $R" # E04: DeleteMessage с неверным ReceiptHandle echo "--- E04 DeleteMessage invalid receipt" R=$(sqspost "Action=DeleteMessage&QueueUrl=${ERR2_URL}&ReceiptHandle=totally-invalid-receipt-xyz&Version=2012-11-05") echo "$R" | grep -qi "error\|ReceiptHandle\|invalid" && ok E04 || fail E04 "нет ошибки: $R" # E05: GetQueueUrl несуществующей очереди echo "--- E05 GetQueueUrl non-existent" R=$(sqs "Action=GetQueueUrl&QueueName=absolutely-not-exist-xyz-2026&Version=2012-11-05") echo "$R" | grep -qi "error\|NonExistent" && ok E05 || fail E05 "нет ошибки: $R" # E06: DeleteMessage уже удалённого (дублированный delete) echo "--- E06 DeleteMessage already deleted (idempotent?)" sqs "Action=CreateQueue&QueueName=t-err06&Version=2012-11-05" > /dev/null ERR6_URL=$(sqs "Action=GetQueueUrl&QueueName=t-err06&Version=2012-11-05" | grep -oP "(?<=)[^<]+") sqs "Action=SendMessage&QueueUrl=${ERR6_URL}&MessageBody=test&Version=2012-11-05" > /dev/null R_RECV=$(sqs "Action=ReceiveMessage&QueueUrl=${ERR6_URL}&MaxNumberOfMessages=1&Version=2012-11-05") RCPT6=$(echo "$R_RECV" | grep -oP "(?<=)[^<]+" | head -1) ENC6=$(encode_receipt "$RCPT6") sqspost "Action=DeleteMessage&QueueUrl=${ERR6_URL}&ReceiptHandle=${ENC6}&Version=2012-11-05" > /dev/null R=$(sqspost "Action=DeleteMessage&QueueUrl=${ERR6_URL}&ReceiptHandle=${ENC6}&Version=2012-11-05") echo "$R" | grep -qi "error\|invalid\|ResponseMetadata" && ok "E06 (double-delete handled)" || warn E06 "$R" # E07: Неверные credentials echo "--- E07 Wrong credentials" R=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "FAKEKEY123:FAKESECRET456" \ "$EP/?Action=ListQueues&Version=2012-11-05") echo "$R" | grep -qi "error\|Unauthorized\|Signature" \ && ok E07 \ || warn E07 "ElasticMQ не проверяет подпись (known limit в strict mode)" # E08: SendMessage с пустым телом echo "--- E08 SendMessage empty body" R=$(sqspost "Action=SendMessage&QueueUrl=${ERR2_URL}&MessageBody=&Version=2012-11-05") echo "$R" | grep -qi "error\|empty\|invalid" && ok E08 || warn E08 "empty body принято: $R" # E09: SendMessage oversized (>256KB) echo "--- E09 SendMessage oversized 300KB" BIG=$(python3 -c "print('X'*307200)" 2>/dev/null) R=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" \ -X POST -H "Content-Type: application/x-www-form-urlencoded" \ --data-urlencode "Action=SendMessage" \ --data-urlencode "QueueUrl=${ERR2_URL}" \ --data-urlencode "MessageBody=${BIG}" \ --data-urlencode "Version=2012-11-05" \ "$EP/") echo "$R" | grep -qi "error\|MessageTooLong\|too large" && ok E09 || warn E09 "oversized принято" # ───────────────────────────────────────────── hdr "PHASE 3 — Продвинутые фичи (A01–A09)" # ───────────────────────────────────────────── # Создаём очередь с VisibilityTimeout=5s для тестов sqs "Action=CreateQueue&QueueName=t-adv&Version=2012-11-05" > /dev/null ADV_URL=$(sqs "Action=GetQueueUrl&QueueName=t-adv&Version=2012-11-05" | grep -oP "(?<=)[^<]+") sqs "Action=SetQueueAttributes&QueueUrl=${ADV_URL}&Attribute.1.Name=VisibilityTimeout&Attribute.1.Value=5&Version=2012-11-05" > /dev/null # A01: VisibilityTimeout — получили не удалили → сообщение возвращается echo "--- A01 VisibilityTimeout (5s)" sqs "Action=SendMessage&QueueUrl=${ADV_URL}&MessageBody=will-return&Version=2012-11-05" > /dev/null R=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=1&Version=2012-11-05") BODY_A01=$(echo "$R" | grep -oP "(?<=)[^<]+" | head -1) RCPT_A01=$(echo "$R" | grep -oP "(?<=)[^<]+" | head -1) echo " Взяли: '$BODY_A01', ждём 6с возврата..." sleep 6 R2=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=1&WaitTimeSeconds=0&Version=2012-11-05") BODY_A01_2=$(echo "$R2" | grep -oP "(?<=)[^<]+" | head -1) [ "$BODY_A01_2" = "will-return" ] && ok "A01 (вернулось: '$BODY_A01_2')" || fail A01 "не вернулось: '$BODY_A01_2'" # A02: ChangeMessageVisibility → немедленная доступность echo "--- A02 ChangeMessageVisibility → 0" RCPT_A02=$(echo "$R2" | grep -oP "(?<=)[^<]+" | head -1) ENC_A02=$(encode_receipt "$RCPT_A02") R=$(sqspost "Action=ChangeMessageVisibility&QueueUrl=${ADV_URL}&ReceiptHandle=${ENC_A02}&VisibilityTimeout=0&Version=2012-11-05") echo "$R" | grep -q "ResponseMetadata" && ok A02 || warn A02 "$R" # A03: SendMessageBatch (10 сообщений) echo "--- A03 SendMessageBatch 10" BATCH="Action=SendMessageBatch&Version=2012-11-05&QueueUrl=${ADV_URL}" for i in $(seq 1 10); do BATCH="${BATCH}&SendMessageBatchRequestEntry.${i}.Id=m${i}&SendMessageBatchRequestEntry.${i}.MessageBody=batch-${i}" done R=$(sqspost "$BATCH") CNT=$(echo "$R" | grep -o "" | wc -l) [ "$CNT" -ge 10 ] && ok "A03 (sent=$CNT)" || fail A03 "sent=$CNT: $R" # A04: ReceiveMessage 10 за раз echo "--- A04 ReceiveMessageBatch MaxNumberOfMessages=10" sleep 1 R=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=10&Version=2012-11-05") RECV=$(echo "$R" | grep -o "" | wc -l) [ "$RECV" -ge 9 ] && ok "A04 (recv=$RECV)" || warn A04 "recv=$RECV из 10+" # Сохраняем для batch delete BATCH_RCPTS=$(echo "$R" | grep -oP "(?<=)[^<]+") # A05: DeleteMessageBatch echo "--- A05 DeleteMessageBatch" BDEL="Action=DeleteMessageBatch&Version=2012-11-05&QueueUrl=${ADV_URL}" IDX=1 while IFS= read -r r; do [ -z "$r" ] && continue ENC=$(encode_receipt "$r") BDEL="${BDEL}&DeleteMessageBatchRequestEntry.${IDX}.Id=d${IDX}&DeleteMessageBatchRequestEntry.${IDX}.ReceiptHandle=${ENC}" IDX=$((IDX+1)) done <<< "$BATCH_RCPTS" R=$(sqspost "$BDEL") echo "$R" | grep -q "DeleteMessageBatchResponse\|ResultCode\|ResponseMetadata" \ && ok "A05 (deleted=$((IDX-1)))" || fail A05 "$R" # A06: Long polling (WaitTimeSeconds=3, пустая очередь) echo "--- A06 Long polling 3s (пустая очередь)" # Дожидаемся что очередь пуста (visibility timeout истёк) sleep 6 START=$(date +%s) R=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=1&WaitTimeSeconds=3&Version=2012-11-05") END=$(date +%s) WAITED=$((END-START)) [ "$WAITED" -ge 2 ] && ok "A06 (waited=${WAITED}s)" || warn A06 "слишком быстро: ${WAITED}s" # A07: MessageAttributes echo "--- A07 MessageAttributes" R=$(sqspost "Action=SendMessage&QueueUrl=${ADV_URL}&MessageBody=with-attrs&Version=2012-11-05\ &MessageAttribute.1.Name=Color&MessageAttribute.1.Value.StringValue=Blue&MessageAttribute.1.Value.DataType=String\ &MessageAttribute.2.Name=Priority&MessageAttribute.2.Value.StringValue=High&MessageAttribute.2.Value.DataType=String") if echo "$R" | grep -q "MessageId"; then R2=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=1&MessageAttributeName.1=Color&MessageAttributeName.2=Priority&Version=2012-11-05") echo "$R2" | grep -qi "Color\|Blue" && ok A07 || warn A07 "атрибуты не вернулись: $R2" else fail A07 "SendMessage с атрибутами: $R" fi # A08: PurgeQueue echo "--- A08 PurgeQueue" for i in $(seq 1 5); do sqs "Action=SendMessage&QueueUrl=${ADV_URL}&MessageBody=purge$i&Version=2012-11-05" > /dev/null; done R=$(sqs "Action=PurgeQueue&QueueUrl=${ADV_URL}&Version=2012-11-05") echo "$R" | grep -q "ResponseMetadata" && ok A08 || fail A08 "$R" sleep 1 CNT_AFTER=$(sqs "Action=GetQueueAttributes&QueueUrl=${ADV_URL}&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05" | grep -oP "(?<=)[^<]+" | head -1) echo " Сообщений после Purge: $CNT_AFTER" # A09: Dead Letter Queue (RedrivePolicy) echo "--- A09 Dead Letter Queue" sqs "Action=CreateQueue&QueueName=t-dlq&Version=2012-11-05" > /dev/null DLQ_URL=$(sqs "Action=GetQueueUrl&QueueName=t-dlq&Version=2012-11-05" | grep -oP "(?<=)[^<]+") DLQ_ARN=$(sqs "Action=GetQueueAttributes&QueueUrl=${DLQ_URL}&AttributeName.1=QueueArn&Version=2012-11-05" | grep -oP "(?<=)[^<]+" | head -1) echo " DLQ ARN: $DLQ_ARN" if [ -n "$DLQ_ARN" ]; then REDRIVE=$(python3 -c "import urllib.parse,json; print(urllib.parse.quote(json.dumps({'deadLetterTargetArn':'$DLQ_ARN','maxReceiveCount':'1'})))" 2>/dev/null) R=$(sqspost "Action=CreateQueue&QueueName=t-src-dlq&Version=2012-11-05&Attribute.1.Name=RedrivePolicy&Attribute.1.Value=${REDRIVE}") echo "$R" | grep -q "QueueUrl\|CreateQueueResponse" && ok A09 || warn A09 "DLQ не принят: $R" else warn A09 "Не получил DLQ ARN" fi # ───────────────────────────────────────────── hdr "PHASE 4 — Multi-tenant изоляция (MT01–MT05)" # ───────────────────────────────────────────── # MT01: Создаём второй тенант echo "--- MT01 Создаём QueueService tenant002" kubectl apply -f - <<'EOF' 2>&1 apiVersion: sqs.kube5s.ru/v1alpha1 kind: QueueService metadata: name: test-tenant-002 namespace: sqs-operator-system spec: tenantId: "test002" memoryMB: 64 storageMB: 512 persistence: false EOF echo " Ждём Ready (до 90с)..." RESULT=$(wait_qs_ready "test-tenant-002" "sqs-operator-system" 45) [ "$RESULT" = "Ready" ] && ok "MT01 (tenant002 Ready)" || fail MT01 "Phase=$RESULT" AK2=$(kubectl -n sless-fn-test002 get secret sqs-creds-test002 -o jsonpath='{.data.accessKey}' | base64 -d 2>/dev/null) SK2=$(kubectl -n sless-fn-test002 get secret sqs-creds-test002 -o jsonpath='{.data.secretKey}' | base64 -d 2>/dev/null) EP2="https://sqs.kube5s.ru/sqs/test002" [ -n "$AK2" ] && ok "MT01b creds OK (AK2=$AK2)" || fail MT01b "нет credentials" # MT02: Одинаковое имя очереди — разные URL echo "--- MT02 Одинаковое имя в разных тенантах" curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" "$EP/?Action=CreateQueue&QueueName=shared-q&Version=2012-11-05" > /dev/null curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" "$EP2/?Action=CreateQueue&QueueName=shared-q&Version=2012-11-05" > /dev/null URL_T1=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" "$EP/?Action=GetQueueUrl&QueueName=shared-q&Version=2012-11-05" | grep -oP "(?<=)[^<]+") URL_T2=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" "$EP2/?Action=GetQueueUrl&QueueName=shared-q&Version=2012-11-05" | grep -oP "(?<=)[^<]+") echo " T1 URL: $URL_T1" echo " T2 URL: $URL_T2" [ -n "$URL_T1" ] && [ -n "$URL_T2" ] && [ "$URL_T1" != "$URL_T2" ] \ && ok MT02 || warn MT02 "URL одинаковые или пустые: $URL_T1 | $URL_T2" # MT03: Cross-tenant message isolation echo "--- MT03 Cross-tenant message isolation" curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" \ "$EP/?Action=SendMessage&QueueUrl=${URL_T1}&MessageBody=tenant001-only-secret&Version=2012-11-05" > /dev/null # Tenant002 пытается прочитать ПРЯМОЙ URL tenant001 R=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" \ "$EP/?Action=ReceiveMessage&QueueUrl=${URL_T1}&MaxNumberOfMessages=1&Version=2012-11-05") BODY_CROSS=$(echo "$R" | grep -oP "(?<=)[^<]+" | head -1) [ "$BODY_CROSS" = "tenant001-only-secret" ] \ && fail MT03 "ISOLATION BREACH: tenant002 прочитал tenant001 сообщение!" \ || ok "MT03 (изоляция работает — тенант002 не прочитал: '$BODY_CROSS')" # MT04: Tenant имеет доступ только к своим очередям echo "--- MT04 Независимые операции в tenant002" curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" \ "$EP2/?Action=SendMessage&QueueUrl=${URL_T2}&MessageBody=t2-msg&Version=2012-11-05" > /dev/null R=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" \ "$EP2/?Action=ReceiveMessage&QueueUrl=${URL_T2}&MaxNumberOfMessages=1&Version=2012-11-05") BODY_T2=$(echo "$R" | grep -oP "(?<=)[^<]+" | head -1) [ "$BODY_T2" = "t2-msg" ] && ok "MT04 (tenant002 работает независимо)" || fail MT04 "body=$BODY_T2" # MT05: Удаляем tenant002 — ресурсы должны очиститься echo "--- MT05 Удаляем tenant002" kubectl delete queueservice test-tenant-002 -n sqs-operator-system 2>&1 | head -1 echo " Ждём удаления ресурсов..." sleep 15 NS=$(kubectl get ns sless-fn-test002 -o name 2>/dev/null) [ -z "$NS" ] && ok "MT05 (namespace удалён)" || warn MT05 "namespace ещё существует: $NS" # ───────────────────────────────────────────── hdr "PHASE 5 — Operator Self-Healing (SH01–SH04)" # ───────────────────────────────────────────── # SH01: Ручное удаление Deployment → оператор пересоздаёт echo "--- SH01 Удаляем Deployment ElasticMQ вручную" kubectl -n sless-fn-test001 delete deployment -l sqs.kube5s.ru/tenant=test001 2>&1 | head -1 echo " Ждём пересоздания оператором..." for i in $(seq 1 30); do READY=$(kubectl -n sless-fn-test001 get deployment -l sqs.kube5s.ru/tenant=test001 -o jsonpath='{.items[0].status.readyReplicas}' 2>/dev/null) [ "$READY" = "1" ] && break sleep 2 done [ "$READY" = "1" ] && ok "SH01 Deployment пересоздан" || fail SH01 "не восстановился (readyReplicas=$READY)" # SH02: Ручное удаление Service echo "--- SH02 Удаляем Service вручную" kubectl -n sless-fn-test001 delete service sqs-svc-test001 2>&1 | head -1 sleep 3 for i in $(seq 1 20); do SVC=$(kubectl -n sless-fn-test001 get service sqs-svc-test001 -o name 2>/dev/null) [ -n "$SVC" ] && break sleep 2 done [ -n "$SVC" ] && ok "SH02 Service пересоздан" || fail SH02 "Service не восстановился" # SH03: Ручное удаление ConfigMap echo "--- SH03 Удаляем ConfigMap вручную" kubectl -n sless-fn-test001 delete configmap sqs-cfg-test001 2>&1 | head -1 sleep 3 for i in $(seq 1 20); do CM=$(kubectl -n sless-fn-test001 get configmap sqs-cfg-test001 -o name 2>/dev/null) [ -n "$CM" ] && break sleep 2 done [ -n "$CM" ] && ok "SH03 ConfigMap пересоздан" || warn SH03 "ConfigMap не пересоздан (оператор может не watch-ить CM)" # SH04: Дожидаемся полного восстановления pod echo "--- SH04 Ждём RestoreReady после self-healing" kubectl -n sless-fn-test001 wait --for=condition=Ready pod -l sqs.kube5s.ru/tenant=test001 --timeout=90s 2>&1 | head -1 sleep 5 PHASE=$(kubectl get queueservice test-tenant-001 -n sqs-operator-system -o jsonpath='{.status.phase}' 2>/dev/null) R=$(sqs "Action=ListQueues&Version=2012-11-05") echo "$R" | grep -q "ListQueuesResponse" \ && ok "SH04 SQS API доступен после self-healing (Phase=$PHASE)" \ || fail SH04 "SQS недоступен: $R" # ───────────────────────────────────────────── hdr "PHASE 6 — Стресс-марафон (20 минут)" # ───────────────────────────────────────────── # Создаём очереди для марафона for q in marathon-a marathon-b marathon-c; do sqs "Action=CreateQueue&QueueName=$q&Version=2012-11-05" > /dev/null done MARATHON_START=$(date +%s) MARATHON_SEC=1200 # 20 минут ITER=0; MERR=0; MSEND=0; MRECV=0; MDEL=0 LAST_REPORT=$(date +%s) echo " Старт: $(date '+%H:%M:%S') → конец в $(date -d "+${MARATHON_SEC} seconds" '+%H:%M:%S' 2>/dev/null || date -v+${MARATHON_SEC}S '+%H:%M:%S' 2>/dev/null)" echo " Режимы: Send, Batch-Send, Receive, Batch-Delete, PurgeQueue, GetAttrs, ErrorCases" while true; do NOW=$(date +%s) [ $((NOW - MARATHON_START)) -ge $MARATHON_SEC ] && break ITER=$((ITER+1)) # Ротация очередей по итерациям case $((ITER % 3)) in 0) Q="marathon-a" ;; 1) Q="marathon-b" ;; 2) Q="marathon-c" ;; esac Q_URL="${EP}/test001/${Q}" # === Обычный send === R=$(sqs "Action=SendMessage&QueueUrl=${Q_URL}&MessageBody=iter${ITER}-$(date +%s)&Version=2012-11-05") if echo "$R" | grep -q "MessageId"; then MSEND=$((MSEND+1)); else MERR=$((MERR+1)); fi # === Batch send каждые 5 итераций === if [ $((ITER % 5)) -eq 0 ]; then BATCH="Action=SendMessageBatch&Version=2012-11-05&QueueUrl=${Q_URL}" for j in $(seq 1 5); do BATCH="${BATCH}&SendMessageBatchRequestEntry.${j}.Id=b${j}&SendMessageBatchRequestEntry.${j}.MessageBody=batch-${ITER}-${j}" done R=$(sqspost "$BATCH") echo "$R" | grep -q "SendMessageBatchResponse\|Id>" || MERR=$((MERR+1)) fi # === Receive + batch delete каждые 3 итерации === if [ $((ITER % 3)) -eq 0 ]; then R=$(sqs "Action=ReceiveMessage&QueueUrl=${Q_URL}&MaxNumberOfMessages=10&WaitTimeSeconds=0&Version=2012-11-05") CNT_R=$(echo "$R" | grep -o "" | wc -l) MRECV=$((MRECV+CNT_R)) if [ "$CNT_R" -gt 0 ]; then RCPTS=$(echo "$R" | grep -oP "(?<=)[^<]+") BDEL="Action=DeleteMessageBatch&Version=2012-11-05&QueueUrl=${Q_URL}" DX=1 while IFS= read -r rc; do [ -z "$rc" ] && continue EN=$(encode_receipt "$rc") BDEL="${BDEL}&DeleteMessageBatchRequestEntry.${DX}.Id=d${DX}&DeleteMessageBatchRequestEntry.${DX}.ReceiptHandle=${EN}" DX=$((DX+1)) done <<< "$RCPTS" R=$(sqspost "$BDEL") echo "$R" | grep -q "DeleteMessageBatchResponse\|ResultCode\|ResponseMetadata" && MDEL=$((MDEL+CNT_R)) || MERR=$((MERR+1)) fi fi # === GetQueueAttributes health check каждые 20 итераций === if [ $((ITER % 20)) -eq 0 ]; then R=$(sqs "Action=GetQueueAttributes&QueueUrl=${Q_URL}&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05") echo "$R" | grep -q "GetQueueAttributesResponse" || MERR=$((MERR+1)) fi # === Намеренные ошибочные запросы каждые 7 итераций === if [ $((ITER % 7)) -eq 0 ]; then # Несуществующая очередь — должна вернуть ошибку, не крашить sqs "Action=ReceiveMessage&QueueUrl=${EP}/test001/no-such-queue-${RANDOM}&Version=2012-11-05" > /dev/null # Дублированное удаление несуществующего сообщения sqspost "Action=DeleteMessage&QueueUrl=${Q_URL}&ReceiptHandle=fake-receipt-${RANDOM}&Version=2012-11-05" > /dev/null fi # === PurgeQueue каждые 100 итераций (держим очереди компактными) === if [ $((ITER % 100)) -eq 0 ]; then for q in marathon-a marathon-b marathon-c; do sqs "Action=PurgeQueue&QueueUrl=${EP}/test001/${q}&Version=2012-11-05" > /dev/null done fi # === Отчёт каждую минуту === if [ $((NOW - LAST_REPORT)) -ge 60 ]; then ELAPSED=$((NOW - MARATHON_START)) MIN=$((ELAPSED/60)); SEC=$((ELAPSED%60)) QA=$(sqs "Action=GetQueueAttributes&QueueUrl=${EP}/test001/marathon-a&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05" | grep -oP "(?<=)[^<]+" | head -1) QB=$(sqs "Action=GetQueueAttributes&QueueUrl=${EP}/test001/marathon-b&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05" | grep -oP "(?<=)[^<]+" | head -1) QC=$(sqs "Action=GetQueueAttributes&QueueUrl=${EP}/test001/marathon-c&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05" | grep -oP "(?<=)[^<]+" | head -1) POD_STATS=$(kubectl -n sless-fn-test001 get pod -l sqs.kube5s.ru/tenant=test001 -o jsonpath='{.items[0].status.containerStatuses[0].restartCount}' 2>/dev/null) printf " [%02d:%02d | iter=%d] send=%d recv=%d del=%d err=%d | q: a=%s b=%s c=%s | pod_restarts=%s\n" \ "$MIN" "$SEC" "$ITER" "$MSEND" "$MRECV" "$MDEL" "$MERR" "$QA" "$QB" "$QC" "$POD_STATS" LAST_REPORT=$NOW fi done MARATHON_END=$(date +%s) MARATHON_ELAPSED=$((MARATHON_END - MARATHON_START)) echo "" echo " Марафон завершён: ${MARATHON_ELAPSED}s, ${ITER} итераций" echo " send=$MSEND recv=$MRECV del=$MDEL errors=$MERR" ERR_RATE=0 [ "$ITER" -gt 0 ] && ERR_RATE=$((MERR * 100 / ITER)) if [ "$MERR" -eq 0 ]; then ok "ST01 Marathon ${ITER}iter, 0 ошибок" elif [ "$ERR_RATE" -lt 1 ]; then ok "ST01 Marathon ${ITER}iter, ${MERR} ошибок (<1%)" else fail ST01 "Marathon: ${MERR}/${ITER} ошибок (${ERR_RATE}%)" fi # ───────────────────────────────────────────── hdr "ИТОГОВЫЙ ОТЧЁТ" # ───────────────────────────────────────────── TOTAL=$((PASS+FAIL+WARN)) echo "" echo " Завершено: $(date '+%Y-%m-%d %H:%M:%S')" echo "" printf " ✅ PASS: %3d\n" "$PASS" printf " ❌ FAIL: %3d\n" "$FAIL" printf " ⚠️ WARN: %3d\n" "$WARN" printf " TOTAL: %3d\n" "$TOTAL" if [ "$FAIL" -gt 0 ]; then echo "" echo " Провалившиеся:" echo -e "$FAIL_LIST" fi if [ "$WARN" -gt 0 ]; then echo "" echo " Предупреждения:" echo -e "$WARN_LIST" fi echo "" echo "==========================================" [ "$FAIL" -eq 0 ] && echo " ✅ ALL PASSED" || echo " ❌ ЕСТЬ ПРОВАЛЫ" echo "==========================================" exit "$FAIL"