560 lines
28 KiB
Bash
Executable File
560 lines
28 KiB
Bash
Executable File
#!/bin/bash
|
||
# test_full_suite.sh — Полный тест-сьют SQS Operator v0.1.4
|
||
# Создан: 2026-04-07
|
||
# Длительность: ~30 минут
|
||
# 6 фаз: базовые, ошибочные, продвинутые, multi-tenant, self-healing, стресс-марафон
|
||
|
||
set -o pipefail
|
||
|
||
# === КОНФИГ ===
|
||
AK=$(kubectl -n sless-fn-test001 get secret sqs-creds-test001 -o jsonpath='{.data.accessKey}' | base64 -d)
|
||
SK=$(kubectl -n sless-fn-test001 get secret sqs-creds-test001 -o jsonpath='{.data.secretKey}' | base64 -d)
|
||
EP="https://sqs.kube5s.ru/sqs/test001"
|
||
|
||
# Счётчики
|
||
PASS=0; FAIL=0; WARN=0
|
||
FAIL_LIST=""
|
||
WARN_LIST=""
|
||
|
||
# === УТИЛИТЫ ===
|
||
ok() { echo " ✅ PASS[$1]"; PASS=$((PASS+1)); }
|
||
fail() { echo " ❌ FAIL[$1]: $2"; FAIL=$((FAIL+1)); FAIL_LIST="${FAIL_LIST}\n - $1: $2"; }
|
||
warn() { echo " ⚠️ WARN[$1]: $2"; WARN=$((WARN+1)); WARN_LIST="${WARN_LIST}\n - $1: $2"; }
|
||
hdr() { echo ""; echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; echo " $1"; echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; }
|
||
|
||
# GET-запрос к SQS
|
||
sqs() {
|
||
curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" "$EP/?$1"
|
||
}
|
||
|
||
# POST-запрос (нужен для ReceiptHandle с #)
|
||
sqspost() {
|
||
curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" \
|
||
-X POST -H "Content-Type: application/x-www-form-urlencoded" \
|
||
--data "$1" "$EP/"
|
||
}
|
||
|
||
# Encode # → %23 в ReceiptHandle
|
||
encode_receipt() { echo "$1" | sed 's/#/%23/g'; }
|
||
|
||
# Ждать Phase=Ready у QueueService
|
||
wait_qs_ready() {
|
||
local name="$1" ns="${2:-sqs-operator-system}" max="${3:-60}"
|
||
for i in $(seq 1 $max); do
|
||
local p=$(kubectl get queueservice "$name" -n "$ns" -o jsonpath='{.status.phase}' 2>/dev/null)
|
||
[ "$p" = "Ready" ] && { echo "Ready"; return 0; }
|
||
sleep 2
|
||
done
|
||
echo "Timeout"
|
||
return 1
|
||
}
|
||
|
||
echo "=========================================="
|
||
echo " SQS Operator Full Test Suite v0.1.4"
|
||
echo " $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo " tenant: test001 ep: $EP"
|
||
echo "=========================================="
|
||
echo " AK: $AK"
|
||
[ -z "$AK" ] && { echo "FATAL: нет credentials, выход."; exit 1; }
|
||
|
||
# ─────────────────────────────────────────────
|
||
hdr "PHASE 1 — Базовые операции (T01–T11)"
|
||
# ─────────────────────────────────────────────
|
||
|
||
# T01: ListQueues
|
||
echo "--- T01 ListQueues"
|
||
R=$(sqs "Action=ListQueues&Version=2012-11-05")
|
||
echo "$R" | grep -q "ListQueuesResponse" && ok T01 || fail T01 "нет ListQueuesResponse: $R"
|
||
|
||
# T02: CreateQueue
|
||
echo "--- T02 CreateQueue"
|
||
R=$(sqs "Action=CreateQueue&QueueName=t-basic&Version=2012-11-05")
|
||
BASIC_URL=$(echo "$R" | grep -oP "(?<=<QueueUrl>)[^<]+")
|
||
[ -n "$BASIC_URL" ] && ok "T02 ($BASIC_URL)" || fail T02 "нет QueueUrl: $R"
|
||
|
||
# T03: CreateQueue duplicate (idempotency)
|
||
echo "--- T03 CreateQueue idempotent"
|
||
R=$(sqs "Action=CreateQueue&QueueName=t-basic&Version=2012-11-05")
|
||
URL2=$(echo "$R" | grep -oP "(?<=<QueueUrl>)[^<]+")
|
||
[ "$URL2" = "$BASIC_URL" ] && ok T03 || fail T03 "URL изменился: $BASIC_URL vs $URL2"
|
||
|
||
# T04: GetQueueUrl
|
||
echo "--- T04 GetQueueUrl"
|
||
R=$(sqs "Action=GetQueueUrl&QueueName=t-basic&Version=2012-11-05")
|
||
echo "$R" | grep -q "QueueUrl" && ok T04 || fail T04 "$R"
|
||
|
||
# T05: SendMessage
|
||
echo "--- T05 SendMessage"
|
||
R=$(sqs "Action=SendMessage&QueueUrl=${BASIC_URL}&MessageBody=hello-world&Version=2012-11-05")
|
||
MSG_ID=$(echo "$R" | grep -oP "(?<=<MessageId>)[^<]+")
|
||
[ -n "$MSG_ID" ] && ok "T05 (id=$MSG_ID)" || fail T05 "$R"
|
||
|
||
# T06: ReceiveMessage
|
||
echo "--- T06 ReceiveMessage"
|
||
R=$(sqs "Action=ReceiveMessage&QueueUrl=${BASIC_URL}&MaxNumberOfMessages=1&Version=2012-11-05")
|
||
BODY=$(echo "$R" | grep -oP "(?<=<Body>)[^<]+" | head -1)
|
||
RECEIPT06=$(echo "$R" | grep -oP "(?<=<ReceiptHandle>)[^<]+" | head -1)
|
||
[ "$BODY" = "hello-world" ] && ok "T06 (body=$BODY)" || fail T06 "body=$BODY, R=$R"
|
||
|
||
# T07: DeleteMessage via POST (# в ReceiptHandle)
|
||
echo "--- T07 DeleteMessage"
|
||
ENC=$(encode_receipt "$RECEIPT06")
|
||
R=$(sqspost "Action=DeleteMessage&QueueUrl=${BASIC_URL}&ReceiptHandle=${ENC}&Version=2012-11-05")
|
||
echo "$R" | grep -q "DeleteMessageResponse\|ResponseMetadata" && ok T07 || fail T07 "$R"
|
||
|
||
# T08: ReceiveMessage after delete (пусто)
|
||
echo "--- T08 ReceiveMessage empty"
|
||
sleep 1
|
||
R=$(sqs "Action=ReceiveMessage&QueueUrl=${BASIC_URL}&MaxNumberOfMessages=1&WaitTimeSeconds=0&Version=2012-11-05")
|
||
echo "$R" | grep -q "<Body>" && fail T08 "сообщение ещё есть после delete" || ok T08
|
||
|
||
# T09: GetQueueAttributes (All)
|
||
echo "--- T09 GetQueueAttributes"
|
||
R=$(sqs "Action=GetQueueAttributes&QueueUrl=${BASIC_URL}&AttributeName.1=All&Version=2012-11-05")
|
||
echo "$R" | grep -q "VisibilityTimeout" && ok T09 || fail T09 "$R"
|
||
|
||
# T10: SetQueueAttributes
|
||
echo "--- T10 SetQueueAttributes"
|
||
R=$(sqs "Action=SetQueueAttributes&QueueUrl=${BASIC_URL}&Attribute.1.Name=VisibilityTimeout&Attribute.1.Value=10&Version=2012-11-05")
|
||
echo "$R" | grep -q "ResponseMetadata" && ok T10 || fail T10 "$R"
|
||
|
||
# T11: DeleteQueue
|
||
echo "--- T11 DeleteQueue"
|
||
R=$(sqs "Action=DeleteQueue&QueueUrl=${BASIC_URL}&Version=2012-11-05")
|
||
echo "$R" | grep -q "ResponseMetadata" && ok T11 || fail T11 "$R"
|
||
|
||
# ─────────────────────────────────────────────
|
||
hdr "PHASE 2 — Ошибочные параметры (E01–E09)"
|
||
# ─────────────────────────────────────────────
|
||
|
||
# E01: InvalidQueueName (спецсимволы)
|
||
echo "--- E01 CreateQueue invalid name (spa ces)"
|
||
R=$(sqs "Action=CreateQueue&QueueName=bad%20name%21&Version=2012-11-05")
|
||
echo "$R" | grep -qi "error\|invalid" && ok E01 || warn E01 "elasticmq принял: $R"
|
||
|
||
# E02: VisibilityTimeout > 43200 (12 часов — максимум AWS)
|
||
echo "--- E02 VisibilityTimeout > 43200"
|
||
sqs "Action=CreateQueue&QueueName=t-err02&Version=2012-11-05" > /dev/null
|
||
ERR2_URL=$(sqs "Action=GetQueueUrl&QueueName=t-err02&Version=2012-11-05" | grep -oP "(?<=<QueueUrl>)[^<]+")
|
||
R=$(sqs "Action=SetQueueAttributes&QueueUrl=${ERR2_URL}&Attribute.1.Name=VisibilityTimeout&Attribute.1.Value=99999&Version=2012-11-05")
|
||
echo "$R" | grep -qi "error\|invalid" && ok E02 || warn E02 "accepted invalidTimeout: $R"
|
||
|
||
# E03: ReceiveMessage от несуществующей очереди
|
||
echo "--- E03 ReceiveMessage non-existent queue"
|
||
R=$(sqs "Action=ReceiveMessage&QueueUrl=${EP}/test001/queue-xyz-does-not-exist&Version=2012-11-05")
|
||
echo "$R" | grep -qi "error\|NonExistent\|not exist" && ok E03 || fail E03 "нет ошибки: $R"
|
||
|
||
# E04: DeleteMessage с неверным ReceiptHandle
|
||
echo "--- E04 DeleteMessage invalid receipt"
|
||
R=$(sqspost "Action=DeleteMessage&QueueUrl=${ERR2_URL}&ReceiptHandle=totally-invalid-receipt-xyz&Version=2012-11-05")
|
||
echo "$R" | grep -qi "error\|ReceiptHandle\|invalid" && ok E04 || fail E04 "нет ошибки: $R"
|
||
|
||
# E05: GetQueueUrl несуществующей очереди
|
||
echo "--- E05 GetQueueUrl non-existent"
|
||
R=$(sqs "Action=GetQueueUrl&QueueName=absolutely-not-exist-xyz-2026&Version=2012-11-05")
|
||
echo "$R" | grep -qi "error\|NonExistent" && ok E05 || fail E05 "нет ошибки: $R"
|
||
|
||
# E06: DeleteMessage уже удалённого (дублированный delete)
|
||
echo "--- E06 DeleteMessage already deleted (idempotent?)"
|
||
sqs "Action=CreateQueue&QueueName=t-err06&Version=2012-11-05" > /dev/null
|
||
ERR6_URL=$(sqs "Action=GetQueueUrl&QueueName=t-err06&Version=2012-11-05" | grep -oP "(?<=<QueueUrl>)[^<]+")
|
||
sqs "Action=SendMessage&QueueUrl=${ERR6_URL}&MessageBody=test&Version=2012-11-05" > /dev/null
|
||
R_RECV=$(sqs "Action=ReceiveMessage&QueueUrl=${ERR6_URL}&MaxNumberOfMessages=1&Version=2012-11-05")
|
||
RCPT6=$(echo "$R_RECV" | grep -oP "(?<=<ReceiptHandle>)[^<]+" | head -1)
|
||
ENC6=$(encode_receipt "$RCPT6")
|
||
sqspost "Action=DeleteMessage&QueueUrl=${ERR6_URL}&ReceiptHandle=${ENC6}&Version=2012-11-05" > /dev/null
|
||
R=$(sqspost "Action=DeleteMessage&QueueUrl=${ERR6_URL}&ReceiptHandle=${ENC6}&Version=2012-11-05")
|
||
echo "$R" | grep -qi "error\|invalid\|ResponseMetadata" && ok "E06 (double-delete handled)" || warn E06 "$R"
|
||
|
||
# E07: Неверные credentials
|
||
echo "--- E07 Wrong credentials"
|
||
R=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "FAKEKEY123:FAKESECRET456" \
|
||
"$EP/?Action=ListQueues&Version=2012-11-05")
|
||
echo "$R" | grep -qi "error\|Unauthorized\|Signature" \
|
||
&& ok E07 \
|
||
|| warn E07 "ElasticMQ не проверяет подпись (known limit в strict mode)"
|
||
|
||
# E08: SendMessage с пустым телом
|
||
echo "--- E08 SendMessage empty body"
|
||
R=$(sqspost "Action=SendMessage&QueueUrl=${ERR2_URL}&MessageBody=&Version=2012-11-05")
|
||
echo "$R" | grep -qi "error\|empty\|invalid" && ok E08 || warn E08 "empty body принято: $R"
|
||
|
||
# E09: SendMessage oversized (>256KB)
|
||
echo "--- E09 SendMessage oversized 300KB"
|
||
BIG=$(python3 -c "print('X'*307200)" 2>/dev/null)
|
||
R=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" \
|
||
-X POST -H "Content-Type: application/x-www-form-urlencoded" \
|
||
--data-urlencode "Action=SendMessage" \
|
||
--data-urlencode "QueueUrl=${ERR2_URL}" \
|
||
--data-urlencode "MessageBody=${BIG}" \
|
||
--data-urlencode "Version=2012-11-05" \
|
||
"$EP/")
|
||
echo "$R" | grep -qi "error\|MessageTooLong\|too large" && ok E09 || warn E09 "oversized принято"
|
||
|
||
# ─────────────────────────────────────────────
|
||
hdr "PHASE 3 — Продвинутые фичи (A01–A09)"
|
||
# ─────────────────────────────────────────────
|
||
|
||
# Создаём очередь с VisibilityTimeout=5s для тестов
|
||
sqs "Action=CreateQueue&QueueName=t-adv&Version=2012-11-05" > /dev/null
|
||
ADV_URL=$(sqs "Action=GetQueueUrl&QueueName=t-adv&Version=2012-11-05" | grep -oP "(?<=<QueueUrl>)[^<]+")
|
||
sqs "Action=SetQueueAttributes&QueueUrl=${ADV_URL}&Attribute.1.Name=VisibilityTimeout&Attribute.1.Value=5&Version=2012-11-05" > /dev/null
|
||
|
||
# A01: VisibilityTimeout — получили не удалили → сообщение возвращается
|
||
echo "--- A01 VisibilityTimeout (5s)"
|
||
sqs "Action=SendMessage&QueueUrl=${ADV_URL}&MessageBody=will-return&Version=2012-11-05" > /dev/null
|
||
R=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=1&Version=2012-11-05")
|
||
BODY_A01=$(echo "$R" | grep -oP "(?<=<Body>)[^<]+" | head -1)
|
||
RCPT_A01=$(echo "$R" | grep -oP "(?<=<ReceiptHandle>)[^<]+" | head -1)
|
||
echo " Взяли: '$BODY_A01', ждём 6с возврата..."
|
||
sleep 6
|
||
R2=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=1&WaitTimeSeconds=0&Version=2012-11-05")
|
||
BODY_A01_2=$(echo "$R2" | grep -oP "(?<=<Body>)[^<]+" | head -1)
|
||
[ "$BODY_A01_2" = "will-return" ] && ok "A01 (вернулось: '$BODY_A01_2')" || fail A01 "не вернулось: '$BODY_A01_2'"
|
||
|
||
# A02: ChangeMessageVisibility → немедленная доступность
|
||
echo "--- A02 ChangeMessageVisibility → 0"
|
||
RCPT_A02=$(echo "$R2" | grep -oP "(?<=<ReceiptHandle>)[^<]+" | head -1)
|
||
ENC_A02=$(encode_receipt "$RCPT_A02")
|
||
R=$(sqspost "Action=ChangeMessageVisibility&QueueUrl=${ADV_URL}&ReceiptHandle=${ENC_A02}&VisibilityTimeout=0&Version=2012-11-05")
|
||
echo "$R" | grep -q "ResponseMetadata" && ok A02 || warn A02 "$R"
|
||
|
||
# A03: SendMessageBatch (10 сообщений)
|
||
echo "--- A03 SendMessageBatch 10"
|
||
BATCH="Action=SendMessageBatch&Version=2012-11-05&QueueUrl=${ADV_URL}"
|
||
for i in $(seq 1 10); do
|
||
BATCH="${BATCH}&SendMessageBatchRequestEntry.${i}.Id=m${i}&SendMessageBatchRequestEntry.${i}.MessageBody=batch-${i}"
|
||
done
|
||
R=$(sqspost "$BATCH")
|
||
CNT=$(echo "$R" | grep -o "<Id>" | wc -l)
|
||
[ "$CNT" -ge 10 ] && ok "A03 (sent=$CNT)" || fail A03 "sent=$CNT: $R"
|
||
|
||
# A04: ReceiveMessage 10 за раз
|
||
echo "--- A04 ReceiveMessageBatch MaxNumberOfMessages=10"
|
||
sleep 1
|
||
R=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=10&Version=2012-11-05")
|
||
RECV=$(echo "$R" | grep -o "<MessageId>" | wc -l)
|
||
[ "$RECV" -ge 9 ] && ok "A04 (recv=$RECV)" || warn A04 "recv=$RECV из 10+"
|
||
# Сохраняем для batch delete
|
||
BATCH_RCPTS=$(echo "$R" | grep -oP "(?<=<ReceiptHandle>)[^<]+")
|
||
|
||
# A05: DeleteMessageBatch
|
||
echo "--- A05 DeleteMessageBatch"
|
||
BDEL="Action=DeleteMessageBatch&Version=2012-11-05&QueueUrl=${ADV_URL}"
|
||
IDX=1
|
||
while IFS= read -r r; do
|
||
[ -z "$r" ] && continue
|
||
ENC=$(encode_receipt "$r")
|
||
BDEL="${BDEL}&DeleteMessageBatchRequestEntry.${IDX}.Id=d${IDX}&DeleteMessageBatchRequestEntry.${IDX}.ReceiptHandle=${ENC}"
|
||
IDX=$((IDX+1))
|
||
done <<< "$BATCH_RCPTS"
|
||
R=$(sqspost "$BDEL")
|
||
echo "$R" | grep -q "DeleteMessageBatchResponse\|ResultCode\|ResponseMetadata" \
|
||
&& ok "A05 (deleted=$((IDX-1)))" || fail A05 "$R"
|
||
|
||
# A06: Long polling (WaitTimeSeconds=3, пустая очередь)
|
||
echo "--- A06 Long polling 3s (пустая очередь)"
|
||
# Дожидаемся что очередь пуста (visibility timeout истёк)
|
||
sleep 6
|
||
START=$(date +%s)
|
||
R=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=1&WaitTimeSeconds=3&Version=2012-11-05")
|
||
END=$(date +%s)
|
||
WAITED=$((END-START))
|
||
[ "$WAITED" -ge 2 ] && ok "A06 (waited=${WAITED}s)" || warn A06 "слишком быстро: ${WAITED}s"
|
||
|
||
# A07: MessageAttributes
|
||
echo "--- A07 MessageAttributes"
|
||
R=$(sqspost "Action=SendMessage&QueueUrl=${ADV_URL}&MessageBody=with-attrs&Version=2012-11-05\
|
||
&MessageAttribute.1.Name=Color&MessageAttribute.1.Value.StringValue=Blue&MessageAttribute.1.Value.DataType=String\
|
||
&MessageAttribute.2.Name=Priority&MessageAttribute.2.Value.StringValue=High&MessageAttribute.2.Value.DataType=String")
|
||
if echo "$R" | grep -q "MessageId"; then
|
||
R2=$(sqs "Action=ReceiveMessage&QueueUrl=${ADV_URL}&MaxNumberOfMessages=1&MessageAttributeName.1=Color&MessageAttributeName.2=Priority&Version=2012-11-05")
|
||
echo "$R2" | grep -qi "Color\|Blue" && ok A07 || warn A07 "атрибуты не вернулись: $R2"
|
||
else
|
||
fail A07 "SendMessage с атрибутами: $R"
|
||
fi
|
||
|
||
# A08: PurgeQueue
|
||
echo "--- A08 PurgeQueue"
|
||
for i in $(seq 1 5); do sqs "Action=SendMessage&QueueUrl=${ADV_URL}&MessageBody=purge$i&Version=2012-11-05" > /dev/null; done
|
||
R=$(sqs "Action=PurgeQueue&QueueUrl=${ADV_URL}&Version=2012-11-05")
|
||
echo "$R" | grep -q "ResponseMetadata" && ok A08 || fail A08 "$R"
|
||
sleep 1
|
||
CNT_AFTER=$(sqs "Action=GetQueueAttributes&QueueUrl=${ADV_URL}&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05" | grep -oP "(?<=<Value>)[^<]+" | head -1)
|
||
echo " Сообщений после Purge: $CNT_AFTER"
|
||
|
||
# A09: Dead Letter Queue (RedrivePolicy)
|
||
echo "--- A09 Dead Letter Queue"
|
||
sqs "Action=CreateQueue&QueueName=t-dlq&Version=2012-11-05" > /dev/null
|
||
DLQ_URL=$(sqs "Action=GetQueueUrl&QueueName=t-dlq&Version=2012-11-05" | grep -oP "(?<=<QueueUrl>)[^<]+")
|
||
DLQ_ARN=$(sqs "Action=GetQueueAttributes&QueueUrl=${DLQ_URL}&AttributeName.1=QueueArn&Version=2012-11-05" | grep -oP "(?<=<Value>)[^<]+" | head -1)
|
||
echo " DLQ ARN: $DLQ_ARN"
|
||
if [ -n "$DLQ_ARN" ]; then
|
||
REDRIVE=$(python3 -c "import urllib.parse,json; print(urllib.parse.quote(json.dumps({'deadLetterTargetArn':'$DLQ_ARN','maxReceiveCount':'1'})))" 2>/dev/null)
|
||
R=$(sqspost "Action=CreateQueue&QueueName=t-src-dlq&Version=2012-11-05&Attribute.1.Name=RedrivePolicy&Attribute.1.Value=${REDRIVE}")
|
||
echo "$R" | grep -q "QueueUrl\|CreateQueueResponse" && ok A09 || warn A09 "DLQ не принят: $R"
|
||
else
|
||
warn A09 "Не получил DLQ ARN"
|
||
fi
|
||
|
||
# ─────────────────────────────────────────────
|
||
hdr "PHASE 4 — Multi-tenant изоляция (MT01–MT05)"
|
||
# ─────────────────────────────────────────────
|
||
|
||
# MT01: Создаём второй тенант
|
||
echo "--- MT01 Создаём QueueService tenant002"
|
||
kubectl apply -f - <<'EOF' 2>&1
|
||
apiVersion: sqs.kube5s.ru/v1alpha1
|
||
kind: QueueService
|
||
metadata:
|
||
name: test-tenant-002
|
||
namespace: sqs-operator-system
|
||
spec:
|
||
tenantId: "test002"
|
||
memoryMB: 64
|
||
storageMB: 512
|
||
persistence: false
|
||
EOF
|
||
|
||
echo " Ждём Ready (до 90с)..."
|
||
RESULT=$(wait_qs_ready "test-tenant-002" "sqs-operator-system" 45)
|
||
[ "$RESULT" = "Ready" ] && ok "MT01 (tenant002 Ready)" || fail MT01 "Phase=$RESULT"
|
||
|
||
AK2=$(kubectl -n sless-fn-test002 get secret sqs-creds-test002 -o jsonpath='{.data.accessKey}' | base64 -d 2>/dev/null)
|
||
SK2=$(kubectl -n sless-fn-test002 get secret sqs-creds-test002 -o jsonpath='{.data.secretKey}' | base64 -d 2>/dev/null)
|
||
EP2="https://sqs.kube5s.ru/sqs/test002"
|
||
[ -n "$AK2" ] && ok "MT01b creds OK (AK2=$AK2)" || fail MT01b "нет credentials"
|
||
|
||
# MT02: Одинаковое имя очереди — разные URL
|
||
echo "--- MT02 Одинаковое имя в разных тенантах"
|
||
curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" "$EP/?Action=CreateQueue&QueueName=shared-q&Version=2012-11-05" > /dev/null
|
||
curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" "$EP2/?Action=CreateQueue&QueueName=shared-q&Version=2012-11-05" > /dev/null
|
||
|
||
URL_T1=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" "$EP/?Action=GetQueueUrl&QueueName=shared-q&Version=2012-11-05" | grep -oP "(?<=<QueueUrl>)[^<]+")
|
||
URL_T2=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" "$EP2/?Action=GetQueueUrl&QueueName=shared-q&Version=2012-11-05" | grep -oP "(?<=<QueueUrl>)[^<]+")
|
||
echo " T1 URL: $URL_T1"
|
||
echo " T2 URL: $URL_T2"
|
||
[ -n "$URL_T1" ] && [ -n "$URL_T2" ] && [ "$URL_T1" != "$URL_T2" ] \
|
||
&& ok MT02 || warn MT02 "URL одинаковые или пустые: $URL_T1 | $URL_T2"
|
||
|
||
# MT03: Cross-tenant message isolation
|
||
echo "--- MT03 Cross-tenant message isolation"
|
||
curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK:$SK" \
|
||
"$EP/?Action=SendMessage&QueueUrl=${URL_T1}&MessageBody=tenant001-only-secret&Version=2012-11-05" > /dev/null
|
||
|
||
# Tenant002 пытается прочитать ПРЯМОЙ URL tenant001
|
||
R=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" \
|
||
"$EP/?Action=ReceiveMessage&QueueUrl=${URL_T1}&MaxNumberOfMessages=1&Version=2012-11-05")
|
||
BODY_CROSS=$(echo "$R" | grep -oP "(?<=<Body>)[^<]+" | head -1)
|
||
[ "$BODY_CROSS" = "tenant001-only-secret" ] \
|
||
&& fail MT03 "ISOLATION BREACH: tenant002 прочитал tenant001 сообщение!" \
|
||
|| ok "MT03 (изоляция работает — тенант002 не прочитал: '$BODY_CROSS')"
|
||
|
||
# MT04: Tenant имеет доступ только к своим очередям
|
||
echo "--- MT04 Независимые операции в tenant002"
|
||
curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" \
|
||
"$EP2/?Action=SendMessage&QueueUrl=${URL_T2}&MessageBody=t2-msg&Version=2012-11-05" > /dev/null
|
||
R=$(curl -sk --aws-sigv4 "aws:amz:us-east-1:sqs" --user "$AK2:$SK2" \
|
||
"$EP2/?Action=ReceiveMessage&QueueUrl=${URL_T2}&MaxNumberOfMessages=1&Version=2012-11-05")
|
||
BODY_T2=$(echo "$R" | grep -oP "(?<=<Body>)[^<]+" | head -1)
|
||
[ "$BODY_T2" = "t2-msg" ] && ok "MT04 (tenant002 работает независимо)" || fail MT04 "body=$BODY_T2"
|
||
|
||
# MT05: Удаляем tenant002 — ресурсы должны очиститься
|
||
echo "--- MT05 Удаляем tenant002"
|
||
kubectl delete queueservice test-tenant-002 -n sqs-operator-system 2>&1 | head -1
|
||
echo " Ждём удаления ресурсов..."
|
||
sleep 15
|
||
NS=$(kubectl get ns sless-fn-test002 -o name 2>/dev/null)
|
||
[ -z "$NS" ] && ok "MT05 (namespace удалён)" || warn MT05 "namespace ещё существует: $NS"
|
||
|
||
# ─────────────────────────────────────────────
|
||
hdr "PHASE 5 — Operator Self-Healing (SH01–SH04)"
|
||
# ─────────────────────────────────────────────
|
||
|
||
# SH01: Ручное удаление Deployment → оператор пересоздаёт
|
||
echo "--- SH01 Удаляем Deployment ElasticMQ вручную"
|
||
kubectl -n sless-fn-test001 delete deployment -l sqs.kube5s.ru/tenant=test001 2>&1 | head -1
|
||
echo " Ждём пересоздания оператором..."
|
||
for i in $(seq 1 30); do
|
||
READY=$(kubectl -n sless-fn-test001 get deployment -l sqs.kube5s.ru/tenant=test001 -o jsonpath='{.items[0].status.readyReplicas}' 2>/dev/null)
|
||
[ "$READY" = "1" ] && break
|
||
sleep 2
|
||
done
|
||
[ "$READY" = "1" ] && ok "SH01 Deployment пересоздан" || fail SH01 "не восстановился (readyReplicas=$READY)"
|
||
|
||
# SH02: Ручное удаление Service
|
||
echo "--- SH02 Удаляем Service вручную"
|
||
kubectl -n sless-fn-test001 delete service sqs-svc-test001 2>&1 | head -1
|
||
sleep 3
|
||
for i in $(seq 1 20); do
|
||
SVC=$(kubectl -n sless-fn-test001 get service sqs-svc-test001 -o name 2>/dev/null)
|
||
[ -n "$SVC" ] && break
|
||
sleep 2
|
||
done
|
||
[ -n "$SVC" ] && ok "SH02 Service пересоздан" || fail SH02 "Service не восстановился"
|
||
|
||
# SH03: Ручное удаление ConfigMap
|
||
echo "--- SH03 Удаляем ConfigMap вручную"
|
||
kubectl -n sless-fn-test001 delete configmap sqs-cfg-test001 2>&1 | head -1
|
||
sleep 3
|
||
for i in $(seq 1 20); do
|
||
CM=$(kubectl -n sless-fn-test001 get configmap sqs-cfg-test001 -o name 2>/dev/null)
|
||
[ -n "$CM" ] && break
|
||
sleep 2
|
||
done
|
||
[ -n "$CM" ] && ok "SH03 ConfigMap пересоздан" || warn SH03 "ConfigMap не пересоздан (оператор может не watch-ить CM)"
|
||
|
||
# SH04: Дожидаемся полного восстановления pod
|
||
echo "--- SH04 Ждём RestoreReady после self-healing"
|
||
kubectl -n sless-fn-test001 wait --for=condition=Ready pod -l sqs.kube5s.ru/tenant=test001 --timeout=90s 2>&1 | head -1
|
||
sleep 5
|
||
PHASE=$(kubectl get queueservice test-tenant-001 -n sqs-operator-system -o jsonpath='{.status.phase}' 2>/dev/null)
|
||
R=$(sqs "Action=ListQueues&Version=2012-11-05")
|
||
echo "$R" | grep -q "ListQueuesResponse" \
|
||
&& ok "SH04 SQS API доступен после self-healing (Phase=$PHASE)" \
|
||
|| fail SH04 "SQS недоступен: $R"
|
||
|
||
# ─────────────────────────────────────────────
|
||
hdr "PHASE 6 — Стресс-марафон (20 минут)"
|
||
# ─────────────────────────────────────────────
|
||
|
||
# Создаём очереди для марафона
|
||
for q in marathon-a marathon-b marathon-c; do
|
||
sqs "Action=CreateQueue&QueueName=$q&Version=2012-11-05" > /dev/null
|
||
done
|
||
|
||
MARATHON_START=$(date +%s)
|
||
MARATHON_SEC=1200 # 20 минут
|
||
ITER=0; MERR=0; MSEND=0; MRECV=0; MDEL=0
|
||
LAST_REPORT=$(date +%s)
|
||
|
||
echo " Старт: $(date '+%H:%M:%S') → конец в $(date -d "+${MARATHON_SEC} seconds" '+%H:%M:%S' 2>/dev/null || date -v+${MARATHON_SEC}S '+%H:%M:%S' 2>/dev/null)"
|
||
echo " Режимы: Send, Batch-Send, Receive, Batch-Delete, PurgeQueue, GetAttrs, ErrorCases"
|
||
|
||
while true; do
|
||
NOW=$(date +%s)
|
||
[ $((NOW - MARATHON_START)) -ge $MARATHON_SEC ] && break
|
||
ITER=$((ITER+1))
|
||
|
||
# Ротация очередей по итерациям
|
||
case $((ITER % 3)) in
|
||
0) Q="marathon-a" ;; 1) Q="marathon-b" ;; 2) Q="marathon-c" ;;
|
||
esac
|
||
Q_URL="${EP}/test001/${Q}"
|
||
|
||
# === Обычный send ===
|
||
R=$(sqs "Action=SendMessage&QueueUrl=${Q_URL}&MessageBody=iter${ITER}-$(date +%s)&Version=2012-11-05")
|
||
if echo "$R" | grep -q "MessageId"; then MSEND=$((MSEND+1)); else MERR=$((MERR+1)); fi
|
||
|
||
# === Batch send каждые 5 итераций ===
|
||
if [ $((ITER % 5)) -eq 0 ]; then
|
||
BATCH="Action=SendMessageBatch&Version=2012-11-05&QueueUrl=${Q_URL}"
|
||
for j in $(seq 1 5); do
|
||
BATCH="${BATCH}&SendMessageBatchRequestEntry.${j}.Id=b${j}&SendMessageBatchRequestEntry.${j}.MessageBody=batch-${ITER}-${j}"
|
||
done
|
||
R=$(sqspost "$BATCH")
|
||
echo "$R" | grep -q "SendMessageBatchResponse\|Id>" || MERR=$((MERR+1))
|
||
fi
|
||
|
||
# === Receive + batch delete каждые 3 итерации ===
|
||
if [ $((ITER % 3)) -eq 0 ]; then
|
||
R=$(sqs "Action=ReceiveMessage&QueueUrl=${Q_URL}&MaxNumberOfMessages=10&WaitTimeSeconds=0&Version=2012-11-05")
|
||
CNT_R=$(echo "$R" | grep -o "<MessageId>" | wc -l)
|
||
MRECV=$((MRECV+CNT_R))
|
||
if [ "$CNT_R" -gt 0 ]; then
|
||
RCPTS=$(echo "$R" | grep -oP "(?<=<ReceiptHandle>)[^<]+")
|
||
BDEL="Action=DeleteMessageBatch&Version=2012-11-05&QueueUrl=${Q_URL}"
|
||
DX=1
|
||
while IFS= read -r rc; do
|
||
[ -z "$rc" ] && continue
|
||
EN=$(encode_receipt "$rc")
|
||
BDEL="${BDEL}&DeleteMessageBatchRequestEntry.${DX}.Id=d${DX}&DeleteMessageBatchRequestEntry.${DX}.ReceiptHandle=${EN}"
|
||
DX=$((DX+1))
|
||
done <<< "$RCPTS"
|
||
R=$(sqspost "$BDEL")
|
||
echo "$R" | grep -q "DeleteMessageBatchResponse\|ResultCode\|ResponseMetadata" && MDEL=$((MDEL+CNT_R)) || MERR=$((MERR+1))
|
||
fi
|
||
fi
|
||
|
||
# === GetQueueAttributes health check каждые 20 итераций ===
|
||
if [ $((ITER % 20)) -eq 0 ]; then
|
||
R=$(sqs "Action=GetQueueAttributes&QueueUrl=${Q_URL}&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05")
|
||
echo "$R" | grep -q "GetQueueAttributesResponse" || MERR=$((MERR+1))
|
||
fi
|
||
|
||
# === Намеренные ошибочные запросы каждые 7 итераций ===
|
||
if [ $((ITER % 7)) -eq 0 ]; then
|
||
# Несуществующая очередь — должна вернуть ошибку, не крашить
|
||
sqs "Action=ReceiveMessage&QueueUrl=${EP}/test001/no-such-queue-${RANDOM}&Version=2012-11-05" > /dev/null
|
||
# Дублированное удаление несуществующего сообщения
|
||
sqspost "Action=DeleteMessage&QueueUrl=${Q_URL}&ReceiptHandle=fake-receipt-${RANDOM}&Version=2012-11-05" > /dev/null
|
||
fi
|
||
|
||
# === PurgeQueue каждые 100 итераций (держим очереди компактными) ===
|
||
if [ $((ITER % 100)) -eq 0 ]; then
|
||
for q in marathon-a marathon-b marathon-c; do
|
||
sqs "Action=PurgeQueue&QueueUrl=${EP}/test001/${q}&Version=2012-11-05" > /dev/null
|
||
done
|
||
fi
|
||
|
||
# === Отчёт каждую минуту ===
|
||
if [ $((NOW - LAST_REPORT)) -ge 60 ]; then
|
||
ELAPSED=$((NOW - MARATHON_START))
|
||
MIN=$((ELAPSED/60)); SEC=$((ELAPSED%60))
|
||
QA=$(sqs "Action=GetQueueAttributes&QueueUrl=${EP}/test001/marathon-a&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05" | grep -oP "(?<=<Value>)[^<]+" | head -1)
|
||
QB=$(sqs "Action=GetQueueAttributes&QueueUrl=${EP}/test001/marathon-b&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05" | grep -oP "(?<=<Value>)[^<]+" | head -1)
|
||
QC=$(sqs "Action=GetQueueAttributes&QueueUrl=${EP}/test001/marathon-c&AttributeName.1=ApproximateNumberOfMessages&Version=2012-11-05" | grep -oP "(?<=<Value>)[^<]+" | head -1)
|
||
POD_STATS=$(kubectl -n sless-fn-test001 get pod -l sqs.kube5s.ru/tenant=test001 -o jsonpath='{.items[0].status.containerStatuses[0].restartCount}' 2>/dev/null)
|
||
printf " [%02d:%02d | iter=%d] send=%d recv=%d del=%d err=%d | q: a=%s b=%s c=%s | pod_restarts=%s\n" \
|
||
"$MIN" "$SEC" "$ITER" "$MSEND" "$MRECV" "$MDEL" "$MERR" "$QA" "$QB" "$QC" "$POD_STATS"
|
||
LAST_REPORT=$NOW
|
||
fi
|
||
done
|
||
|
||
MARATHON_END=$(date +%s)
|
||
MARATHON_ELAPSED=$((MARATHON_END - MARATHON_START))
|
||
echo ""
|
||
echo " Марафон завершён: ${MARATHON_ELAPSED}s, ${ITER} итераций"
|
||
echo " send=$MSEND recv=$MRECV del=$MDEL errors=$MERR"
|
||
|
||
ERR_RATE=0
|
||
[ "$ITER" -gt 0 ] && ERR_RATE=$((MERR * 100 / ITER))
|
||
|
||
if [ "$MERR" -eq 0 ]; then
|
||
ok "ST01 Marathon ${ITER}iter, 0 ошибок"
|
||
elif [ "$ERR_RATE" -lt 1 ]; then
|
||
ok "ST01 Marathon ${ITER}iter, ${MERR} ошибок (<1%)"
|
||
else
|
||
fail ST01 "Marathon: ${MERR}/${ITER} ошибок (${ERR_RATE}%)"
|
||
fi
|
||
|
||
# ─────────────────────────────────────────────
|
||
hdr "ИТОГОВЫЙ ОТЧЁТ"
|
||
# ─────────────────────────────────────────────
|
||
|
||
TOTAL=$((PASS+FAIL+WARN))
|
||
echo ""
|
||
echo " Завершено: $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo ""
|
||
printf " ✅ PASS: %3d\n" "$PASS"
|
||
printf " ❌ FAIL: %3d\n" "$FAIL"
|
||
printf " ⚠️ WARN: %3d\n" "$WARN"
|
||
printf " TOTAL: %3d\n" "$TOTAL"
|
||
|
||
if [ "$FAIL" -gt 0 ]; then
|
||
echo ""
|
||
echo " Провалившиеся:"
|
||
echo -e "$FAIL_LIST"
|
||
fi
|
||
if [ "$WARN" -gt 0 ]; then
|
||
echo ""
|
||
echo " Предупреждения:"
|
||
echo -e "$WARN_LIST"
|
||
fi
|
||
|
||
echo ""
|
||
echo "=========================================="
|
||
[ "$FAIL" -eq 0 ] && echo " ✅ ALL PASSED" || echo " ❌ ЕСТЬ ПРОВАЛЫ"
|
||
echo "=========================================="
|
||
exit "$FAIL"
|