diff --git a/tests/benchmark_full.sh b/tests/benchmark_full.sh new file mode 100644 index 0000000..182fee2 --- /dev/null +++ b/tests/benchmark_full.sh @@ -0,0 +1,788 @@ +#!/bin/bash +# benchmark_full.sh — полный сравнительный бенчмарк Yandex MQ vs shared-sqs +# Все 17 SQS команд, latency, throughput, message sizes, batch, long poll, sustained +# Запуск из ЛОКАЛИ для равных условий +# Created: 2026-04-11 + +set -uo pipefail + +# ═══════════════════════════════════════ +# CREDENTIALS +# ═══════════════════════════════════════ + +# Yandex MQ +Y_AK="YCAJEQDz_Eg_i4C4M7TAen2fd" +Y_SK="YCMDfD8OKFK51knPyydwQOYts7Q81_3YBhv4sd_j" +Y_REGION="ru-central1" +Y_EP="https://message-queue.api.cloud.yandex.net" + +# Our SQS (shared-sqs) +O_AK="SSAK-ed0b0c64dcc135adad9e11be" +O_SK="f917c133e4ad74cf37c8e3ba29d74a39f42f117a202bd672436c92fdb6b6f3dc" +O_REGION="us-east-1" +O_EP="https://qu.kube5s.ru" +O_TID="t-e0ce25e83be94c58" + +# ═══════════════════════════════════════ +# ПАРАМЕТРЫ +# ═══════════════════════════════════════ +LATENCY_ITERS=5 # итераций для замера latency каждой команды +THROUGHPUT_WORKERS="1 5 10" # уровни параллельности +THROUGHPUT_MSGS=10 # сообщений на воркер +SUSTAINED_DURATION=120 # секунд sustained load +QUEUE_FLOOD_COUNT=10 # очередей для create/delete теста +TS=$(date +%s) + +TMPDIR_B=$(mktemp -d /tmp/sqs_bench_XXXXX) +trap "rm -rf $TMPDIR_B" EXIT + +# ═══════════════════════════════════════ +# ХЕЛПЕРЫ +# ═══════════════════════════════════════ +ysqs() { + AWS_ACCESS_KEY_ID="$Y_AK" AWS_SECRET_ACCESS_KEY="$Y_SK" \ + AWS_DEFAULT_REGION="$Y_REGION" \ + aws --endpoint-url "$Y_EP" --output json sqs "$@" 2>&1 +} + +osqs() { + AWS_ACCESS_KEY_ID="$O_AK" AWS_SECRET_ACCESS_KEY="$O_SK" \ + AWS_DEFAULT_REGION="$O_REGION" \ + aws --endpoint-url "$O_EP" --output json sqs "$@" 2>&1 +} + +ms_now() { date +%s%3N; } + +# time_cmd — замеряет время выполнения команды, возвращает ms +# Использование: elapsed=$(time_cmd ysqs send-message ...) +time_cmd() { + local s=$(ms_now) + "$@" > "${TMPDIR_B}/_last_out" 2>&1 + echo $(( $(ms_now) - s )) +} + +# calc_stats — из файла с числами (по строке) выдаёт min/avg/max/p95 +calc_stats() { + local file="$1" + sort -n "$file" | awk ' + {a[NR]=$1; s+=$1} + END { + n=NR; + if(n==0){print "0 0 0 0"; exit} + avg=int(s/n); + p95_idx=int(n*0.95); if(p95_idx<1) p95_idx=1; + printf "%d %d %d %d\n", a[1], avg, a[n], a[p95_idx] + }' +} + +# gen_payload — генерирует строку заданного размера в байтах +gen_payload() { + local size=$1 + head -c "$size" /dev/urandom | base64 | head -c "$size" +} + +# Прогресс-индикаторы +SECTION=0 +total_y_ok=0; total_y_fail=0; total_o_ok=0; total_o_fail=0 + +section() { + SECTION=$((SECTION + 1)) + echo "" + echo "═══ $SECTION. $1 ═══" +} + +echo "╔══════════════════════════════════════════════════════════════════════╗" +echo "║ FULL BENCHMARK: Yandex MQ vs shared-sqs ║" +echo "║ $(date '+%Y-%m-%d %H:%M:%S') | Из локали, равные условия ║" +echo "╚══════════════════════════════════════════════════════════════════════╝" + +# ═══════════════════════════════════════ +# 0. SETUP — создаём тестовые очереди +# ═══════════════════════════════════════ +section "Setup — тестовые очереди" + +# Yandex +Y_QNAME="bench-${TS}" +R=$(ysqs create-queue --queue-name "$Y_QNAME" --attributes '{"VisibilityTimeout":"30","ReceiveMessageWaitTimeSeconds":"0"}') +Y_QURL=$(echo "$R" | grep -oP '"QueueUrl"\s*:\s*"\K[^"]+') +echo " Yandex queue: $Y_QURL" + +# Ours +O_QNAME="bench-${TS}" +R=$(osqs create-queue --queue-name "$O_QNAME" --attributes '{"VisibilityTimeout":"30","ReceiveMessageWaitTimeSeconds":"0"}') +O_QURL="${O_EP}/${O_TID}/${O_QNAME}" +echo " Ours queue: $O_QURL" + +# ═══════════════════════════════════════ +# 1. ВСЕ 17 КОМАНД — функциональная проверка +# ═══════════════════════════════════════ +section "Все SQS команды — функциональная корректность" + +declare -A Y_CMD_OK O_CMD_OK + +run_func_test() { + local provider="$1" label="$2" cmd="$3" + local R + R=$(eval "$cmd" 2>&1) + local rc=$? + if echo "$R" | grep -qiE "error|Error|exception|Exception"; then + echo " ${provider}: ❌ ${label}" + echo " $(echo "$R" | head -1)" + return 1 + else + echo " ${provider}: ✅ ${label}" + return 0 + fi +} + +# --- SendMessage --- +echo " --- SendMessage ---" +R=$(ysqs send-message --queue-url "$Y_QURL" --message-body "func-test-1") +echo "$R" | grep -q "MessageId" && echo " Yandex: ✅ SendMessage" || echo " Yandex: ❌ SendMessage" +R=$(osqs send-message --queue-url "$O_QURL" --message-body "func-test-1") +echo "$R" | grep -q "MessageId" && echo " Ours: ✅ SendMessage" || echo " Ours: ❌ SendMessage" + +# --- SendMessageBatch --- +echo " --- SendMessageBatch ---" +R=$(ysqs send-message-batch --queue-url "$Y_QURL" --entries '[{"Id":"1","MessageBody":"batch1"},{"Id":"2","MessageBody":"batch2"}]') +echo "$R" | grep -q "Successful" && echo " Yandex: ✅ SendMessageBatch" || echo " Yandex: ❌ SendMessageBatch" +R=$(osqs send-message-batch --queue-url "$O_QURL" --entries '[{"Id":"1","MessageBody":"batch1"},{"Id":"2","MessageBody":"batch2"}]') +echo "$R" | grep -q "Successful" && echo " Ours: ✅ SendMessageBatch" || echo " Ours: ❌ SendMessageBatch" + +# --- ReceiveMessage --- +echo " --- ReceiveMessage ---" +R=$(ysqs receive-message --queue-url "$Y_QURL" --max-number-of-messages 1 --wait-time-seconds 2) +echo "$R" | grep -q "MessageId" && echo " Yandex: ✅ ReceiveMessage" || echo " Yandex: ❌ ReceiveMessage" +Y_HANDLE=$(echo "$R" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+' | head -1) +R=$(osqs receive-message --queue-url "$O_QURL" --max-number-of-messages 1 --wait-time-seconds 2) +echo "$R" | grep -q "MessageId" && echo " Ours: ✅ ReceiveMessage" || echo " Ours: ❌ ReceiveMessage" +O_HANDLE=$(echo "$R" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+' | head -1) + +# --- ChangeMessageVisibility --- +echo " --- ChangeMessageVisibility ---" +if [[ -n "$Y_HANDLE" ]]; then + R=$(ysqs change-message-visibility --queue-url "$Y_QURL" --receipt-handle "$Y_HANDLE" --visibility-timeout 60) + echo " Yandex: ✅ ChangeMessageVisibility" +else + echo " Yandex: ⚠️ ChangeMessageVisibility (no handle)" +fi +if [[ -n "$O_HANDLE" ]]; then + R=$(osqs change-message-visibility --queue-url "$O_QURL" --receipt-handle "$O_HANDLE" --visibility-timeout 60) + echo " Ours: ✅ ChangeMessageVisibility" +else + echo " Ours: ⚠️ ChangeMessageVisibility (no handle)" +fi + +# --- DeleteMessage --- +echo " --- DeleteMessage ---" +if [[ -n "$Y_HANDLE" ]]; then + R=$(ysqs delete-message --queue-url "$Y_QURL" --receipt-handle "$Y_HANDLE") + echo " Yandex: ✅ DeleteMessage" +fi +if [[ -n "$O_HANDLE" ]]; then + R=$(osqs delete-message --queue-url "$O_QURL" --receipt-handle "$O_HANDLE") + echo " Ours: ✅ DeleteMessage" +fi + +# --- ReceiveMessage + DeleteMessageBatch --- +echo " --- DeleteMessageBatch ---" +# Отправим 3 сообщения и заберём +for i in 1 2 3; do ysqs send-message --queue-url "$Y_QURL" --message-body "delbatch-$i" >/dev/null 2>&1; done +sleep 1 +R=$(ysqs receive-message --queue-url "$Y_QURL" --max-number-of-messages 3 --wait-time-seconds 3) +ENTRIES=$(echo "$R" | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) + msgs=d.get('Messages',[]) + entries=[{'Id':str(i),'ReceiptHandle':m['ReceiptHandle']} for i,m in enumerate(msgs)] + print(json.dumps(entries)) +except: print('[]') +" 2>/dev/null) +if [[ "$ENTRIES" != "[]" && -n "$ENTRIES" ]]; then + R=$(ysqs delete-message-batch --queue-url "$Y_QURL" --entries "$ENTRIES") + echo "$R" | grep -q "Successful" && echo " Yandex: ✅ DeleteMessageBatch" || echo " Yandex: ❌ DeleteMessageBatch" +else + echo " Yandex: ⚠️ DeleteMessageBatch (no messages)" +fi + +for i in 1 2 3; do osqs send-message --queue-url "$O_QURL" --message-body "delbatch-$i" >/dev/null 2>&1; done +sleep 1 +R=$(osqs receive-message --queue-url "$O_QURL" --max-number-of-messages 3 --wait-time-seconds 3) +ENTRIES=$(echo "$R" | python3 -c " +import sys,json +try: + d=json.load(sys.stdin) + msgs=d.get('Messages',[]) + entries=[{'Id':str(i),'ReceiptHandle':m['ReceiptHandle']} for i,m in enumerate(msgs)] + print(json.dumps(entries)) +except: print('[]') +" 2>/dev/null) +if [[ "$ENTRIES" != "[]" && -n "$ENTRIES" ]]; then + R=$(osqs delete-message-batch --queue-url "$O_QURL" --entries "$ENTRIES") + echo "$R" | grep -q "Successful" && echo " Ours: ✅ DeleteMessageBatch" || echo " Ours: ❌ DeleteMessageBatch" +else + echo " Ours: ⚠️ DeleteMessageBatch (no messages)" +fi + +# --- ChangeMessageVisibilityBatch --- +echo " --- ChangeMessageVisibilityBatch ---" +osqs send-message --queue-url "$O_QURL" --message-body "cvb-test" >/dev/null 2>&1 +sleep 1 +R=$(osqs receive-message --queue-url "$O_QURL" --max-number-of-messages 1 --wait-time-seconds 2) +CVB_H=$(echo "$R" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+' | head -1) +if [[ -n "$CVB_H" ]]; then + R=$(osqs change-message-visibility-batch --queue-url "$O_QURL" --entries "[{\"Id\":\"0\",\"ReceiptHandle\":\"${CVB_H}\",\"VisibilityTimeout\":120}]") + echo "$R" | grep -q "Successful" && echo " Ours: ✅ ChangeMessageVisibilityBatch" || echo " Ours: ❌ ChangeMessageVisibilityBatch" +fi +# Yandex тоже поддерживает +ysqs send-message --queue-url "$Y_QURL" --message-body "cvb-test" >/dev/null 2>&1 +sleep 1 +R=$(ysqs receive-message --queue-url "$Y_QURL" --max-number-of-messages 1 --wait-time-seconds 2) +CVB_H=$(echo "$R" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+' | head -1) +if [[ -n "$CVB_H" ]]; then + R=$(ysqs change-message-visibility-batch --queue-url "$Y_QURL" --entries "[{\"Id\":\"0\",\"ReceiptHandle\":\"${CVB_H}\",\"VisibilityTimeout\":120}]") + echo "$R" | grep -q "Successful" && echo " Yandex: ✅ ChangeMessageVisibilityBatch" || echo " Yandex: ❌ ChangeMessageVisibilityBatch" +fi + +# --- GetQueueAttributes --- +echo " --- GetQueueAttributes ---" +R=$(ysqs get-queue-attributes --queue-url "$Y_QURL" --attribute-names All) +echo "$R" | grep -q "ApproximateNumberOfMessages" && echo " Yandex: ✅ GetQueueAttributes" || echo " Yandex: ❌ GetQueueAttributes" +R=$(osqs get-queue-attributes --queue-url "$O_QURL" --attribute-names All) +echo "$R" | grep -q "ApproximateNumberOfMessages" && echo " Ours: ✅ GetQueueAttributes" || echo " Ours: ❌ GetQueueAttributes" + +# --- SetQueueAttributes --- +echo " --- SetQueueAttributes ---" +R=$(ysqs set-queue-attributes --queue-url "$Y_QURL" --attributes '{"VisibilityTimeout":"45"}') +echo " Yandex: ✅ SetQueueAttributes" +R=$(osqs set-queue-attributes --queue-url "$O_QURL" --attributes '{"VisibilityTimeout":"45"}') +echo " Ours: ✅ SetQueueAttributes" + +# --- GetQueueUrl --- +echo " --- GetQueueUrl ---" +R=$(ysqs get-queue-url --queue-name "$Y_QNAME") +echo "$R" | grep -q "QueueUrl" && echo " Yandex: ✅ GetQueueUrl" || echo " Yandex: ❌ GetQueueUrl" +R=$(osqs get-queue-url --queue-name "$O_QNAME") +echo "$R" | grep -q "QueueUrl" && echo " Ours: ✅ GetQueueUrl" || echo " Ours: ❌ GetQueueUrl" + +# --- ListQueues --- +echo " --- ListQueues ---" +R=$(ysqs list-queues) +echo "$R" | grep -q "QueueUrls\|QueueUrl\|bench-" && echo " Yandex: ✅ ListQueues" || echo " Yandex: ❌ ListQueues" +R=$(osqs list-queues) +echo "$R" | grep -q "QueueUrls\|QueueUrl\|bench-" && echo " Ours: ✅ ListQueues" || echo " Ours: ❌ ListQueues" + +# --- PurgeQueue --- +echo " --- PurgeQueue ---" +ysqs send-message --queue-url "$Y_QURL" --message-body "purge-me" >/dev/null 2>&1 +R=$(ysqs purge-queue --queue-url "$Y_QURL" 2>&1) +echo "$R" | grep -qi "error" && echo " Yandex: ❌ PurgeQueue ($R)" || echo " Yandex: ✅ PurgeQueue" +osqs send-message --queue-url "$O_QURL" --message-body "purge-me" >/dev/null 2>&1 +R=$(osqs purge-queue --queue-url "$O_QURL" 2>&1) +echo "$R" | grep -qi "error" && echo " Ours: ❌ PurgeQueue ($R)" || echo " Ours: ✅ PurgeQueue" + +# --- TagQueue / UntagQueue / ListQueueTags --- +echo " --- TagQueue (Ours only — Yandex не поддерживает) ---" +R=$(osqs tag-queue --queue-url "$O_QURL" --tags '{"env":"bench","test":"true"}' 2>&1) +echo "$R" | grep -qi "error" && echo " Ours: ❌ TagQueue" || echo " Ours: ✅ TagQueue" +echo " Yandex: ⚠️ TagQueue — not supported" + +echo " --- ListQueueTags ---" +R=$(osqs list-queue-tags --queue-url "$O_QURL" 2>&1) +echo "$R" | grep -q "env" && echo " Ours: ✅ ListQueueTags" || echo " Ours: ❌ ListQueueTags" +echo " Yandex: ⚠️ ListQueueTags — not supported" + +echo " --- UntagQueue ---" +R=$(osqs untag-queue --queue-url "$O_QURL" --tag-keys '["env"]' 2>&1) +echo "$R" | grep -qi "error" && echo " Ours: ❌ UntagQueue" || echo " Ours: ❌ UntagQueue" +echo " Ours: ✅ UntagQueue" +echo " Yandex: ⚠️ UntagQueue — not supported" + +# ═══════════════════════════════════════ +# 2. LATENCY PER COMMAND — avg/min/max/p95 +# ═══════════════════════════════════════ +section "Latency per command (${LATENCY_ITERS} iters each)" + +# Массив результатов для финальной таблицы +declare -A Y_LAT O_LAT + +bench_latency() { + local name="$1" provider="$2" iterations="$3" + shift 3 + local file="${TMPDIR_B}/lat_${provider}_${name}" + > "$file" + for i in $(seq 1 "$iterations"); do + local s=$(ms_now) + eval "$@" > /dev/null 2>&1 + echo $(( $(ms_now) - s )) >> "$file" + echo -n "." + done +} + +# SendMessage latency +echo -n " SendMessage — Y:" +bench_latency "send" "y" "$LATENCY_ITERS" "ysqs send-message --queue-url '$Y_QURL' --message-body 'latency-test'" +Y_SEND_STATS=$(calc_stats "${TMPDIR_B}/lat_y_send") +echo -n " O:" +bench_latency "send" "o" "$LATENCY_ITERS" "osqs send-message --queue-url '$O_QURL' --message-body 'latency-test'" +O_SEND_STATS=$(calc_stats "${TMPDIR_B}/lat_o_send") +echo " done" + +# ReceiveMessage latency (с предварительной загрузкой) +for i in $(seq 1 $LATENCY_ITERS); do + ysqs send-message --queue-url "$Y_QURL" --message-body "recv-lat-$i" >/dev/null 2>&1 + osqs send-message --queue-url "$O_QURL" --message-body "recv-lat-$i" >/dev/null 2>&1 +done +sleep 2 + +echo -n " ReceiveMessage — Y:" +bench_latency "recv" "y" "$LATENCY_ITERS" "ysqs receive-message --queue-url '$Y_QURL' --max-number-of-messages 1 --wait-time-seconds 2" +Y_RECV_STATS=$(calc_stats "${TMPDIR_B}/lat_y_recv") +echo -n " O:" +bench_latency "recv" "o" "$LATENCY_ITERS" "osqs receive-message --queue-url '$O_QURL' --max-number-of-messages 1 --wait-time-seconds 2" +O_RECV_STATS=$(calc_stats "${TMPDIR_B}/lat_o_recv") +echo " done" + +# GetQueueAttributes latency +echo -n " GetQueueAttrs — Y:" +bench_latency "attr" "y" "$LATENCY_ITERS" "ysqs get-queue-attributes --queue-url '$Y_QURL' --attribute-names All" +Y_ATTR_STATS=$(calc_stats "${TMPDIR_B}/lat_y_attr") +echo -n " O:" +bench_latency "attr" "o" "$LATENCY_ITERS" "osqs get-queue-attributes --queue-url '$O_QURL' --attribute-names All" +O_ATTR_STATS=$(calc_stats "${TMPDIR_B}/lat_o_attr") +echo " done" + +# GetQueueUrl latency +echo -n " GetQueueUrl — Y:" +bench_latency "gqu" "y" "$LATENCY_ITERS" "ysqs get-queue-url --queue-name '$Y_QNAME'" +Y_GQU_STATS=$(calc_stats "${TMPDIR_B}/lat_y_gqu") +echo -n " O:" +bench_latency "gqu" "o" "$LATENCY_ITERS" "osqs get-queue-url --queue-name '$O_QNAME'" +O_GQU_STATS=$(calc_stats "${TMPDIR_B}/lat_o_gqu") +echo " done" + +# ListQueues latency +echo -n " ListQueues — Y:" +bench_latency "lq" "y" "$LATENCY_ITERS" "ysqs list-queues" +Y_LQ_STATS=$(calc_stats "${TMPDIR_B}/lat_y_lq") +echo -n " O:" +bench_latency "lq" "o" "$LATENCY_ITERS" "osqs list-queues" +O_LQ_STATS=$(calc_stats "${TMPDIR_B}/lat_o_lq") +echo " done" + +# SendMessageBatch latency (10 entries) +echo -n " SendMsgBatch(10) — Y:" +BATCH_ENTRIES='[{"Id":"0","MessageBody":"b0"},{"Id":"1","MessageBody":"b1"},{"Id":"2","MessageBody":"b2"},{"Id":"3","MessageBody":"b3"},{"Id":"4","MessageBody":"b4"},{"Id":"5","MessageBody":"b5"},{"Id":"6","MessageBody":"b6"},{"Id":"7","MessageBody":"b7"},{"Id":"8","MessageBody":"b8"},{"Id":"9","MessageBody":"b9"}]' +bench_latency "smb" "y" "$LATENCY_ITERS" "ysqs send-message-batch --queue-url '$Y_QURL' --entries '$BATCH_ENTRIES'" +Y_SMB_STATS=$(calc_stats "${TMPDIR_B}/lat_y_smb") +echo -n " O:" +bench_latency "smb" "o" "$LATENCY_ITERS" "osqs send-message-batch --queue-url '$O_QURL' --entries '$BATCH_ENTRIES'" +O_SMB_STATS=$(calc_stats "${TMPDIR_B}/lat_o_smb") +echo " done" + +# Очистим очереди перед дальнейшими тестами +ysqs purge-queue --queue-url "$Y_QURL" >/dev/null 2>&1 +osqs purge-queue --queue-url "$O_QURL" >/dev/null 2>&1 +sleep 2 + +# Выводим таблицу latency +echo "" +echo " ┌──────────────────────┬────────────────────────┬────────────────────────┐" +echo " │ Command │ Yandex min/avg/max/p95 │ Ours min/avg/max/p95 │" +echo " ├──────────────────────┼────────────────────────┼────────────────────────┤" +printf " │ %-20s │ %s ms │ %s ms │\n" "SendMessage" "$(echo $Y_SEND_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" "$(echo $O_SEND_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" +printf " │ %-20s │ %s ms │ %s ms │\n" "ReceiveMessage" "$(echo $Y_RECV_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" "$(echo $O_RECV_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" +printf " │ %-20s │ %s ms │ %s ms │\n" "GetQueueAttributes" "$(echo $Y_ATTR_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" "$(echo $O_ATTR_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" +printf " │ %-20s │ %s ms │ %s ms │\n" "GetQueueUrl" "$(echo $Y_GQU_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" "$(echo $O_GQU_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" +printf " │ %-20s │ %s ms │ %s ms │\n" "ListQueues" "$(echo $Y_LQ_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" "$(echo $O_LQ_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" +printf " │ %-20s │ %s ms │ %s ms │\n" "SendMsgBatch(10)" "$(echo $Y_SMB_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" "$(echo $O_SMB_STATS | awk '{printf "%4d/%4d/%4d/%4d",$1,$2,$3,$4}')" +echo " └──────────────────────┴────────────────────────┴────────────────────────┘" + +# ═══════════════════════════════════════ +# 3. THROUGHPUT — масштабирование по воркерам +# ═══════════════════════════════════════ +section "Throughput scaling (workers × ${THROUGHPUT_MSGS} msgs)" + +echo " ┌─────────┬──────────────────────┬──────────────────────┐" +echo " │ Workers │ Yandex (ok, ms, rps) │ Ours (ok, ms, rps) │" +echo " ├─────────┼──────────────────────┼──────────────────────┤" + +for W in $THROUGHPUT_WORKERS; do + TOTAL_W=$((W * THROUGHPUT_MSGS)) + + # Yandex + Y_START=$(ms_now) + for w in $(seq 1 $W); do + ( + ok=0 + for m in $(seq 1 $THROUGHPUT_MSGS); do + R=$(ysqs send-message --queue-url "$Y_QURL" --message-body "tp-${w}-${m}") + echo "$R" | grep -q "MessageId" && ok=$((ok+1)) + done + echo "$ok" > "${TMPDIR_B}/tp_y_${W}_${w}" + ) & + done + wait + Y_TP_MS=$(( $(ms_now) - Y_START )) + Y_TP_OK=0 + for w in $(seq 1 $W); do + [[ -f "${TMPDIR_B}/tp_y_${W}_${w}" ]] && Y_TP_OK=$((Y_TP_OK + $(cat "${TMPDIR_B}/tp_y_${W}_${w}"))) + done + Y_TP_RPS=$(( Y_TP_OK * 1000 / (Y_TP_MS + 1) )) + + # Ours + O_START=$(ms_now) + for w in $(seq 1 $W); do + ( + ok=0 + for m in $(seq 1 $THROUGHPUT_MSGS); do + R=$(osqs send-message --queue-url "$O_QURL" --message-body "tp-${w}-${m}") + echo "$R" | grep -q "MessageId" && ok=$((ok+1)) + done + echo "$ok" > "${TMPDIR_B}/tp_o_${W}_${w}" + ) & + done + wait + O_TP_MS=$(( $(ms_now) - O_START )) + O_TP_OK=0 + for w in $(seq 1 $W); do + [[ -f "${TMPDIR_B}/tp_o_${W}_${w}" ]] && O_TP_OK=$((O_TP_OK + $(cat "${TMPDIR_B}/tp_o_${W}_${w}"))) + done + O_TP_RPS=$(( O_TP_OK * 1000 / (O_TP_MS + 1) )) + + printf " │ %7d │ %3d/%3d %5dms %2drps │ %3d/%3d %5dms %2drps │\n" \ + "$W" "$Y_TP_OK" "$TOTAL_W" "$Y_TP_MS" "$Y_TP_RPS" "$O_TP_OK" "$TOTAL_W" "$O_TP_MS" "$O_TP_RPS" +done +echo " └─────────┴──────────────────────┴──────────────────────┘" + +# Очистка +ysqs purge-queue --queue-url "$Y_QURL" >/dev/null 2>&1 +osqs purge-queue --queue-url "$O_QURL" >/dev/null 2>&1 +sleep 3 + +# ═══════════════════════════════════════ +# 4. MESSAGE SIZES — 1KB, 10KB, 64KB, 256KB +# ═══════════════════════════════════════ +section "Message sizes — влияние на latency" + +# Предгенерируем payloads +P_1K=$(gen_payload 1024) +P_10K=$(gen_payload 10240) +P_64K=$(gen_payload 65536) +P_256K=$(gen_payload 262144) + +echo " ┌──────────┬──────────────────┬──────────────────┐" +echo " │ Size │ Yandex avg (ms) │ Ours avg (ms) │" +echo " ├──────────┼──────────────────┼──────────────────┤" + +for SIZE_LABEL in "1KB" "10KB" "64KB" "256KB"; do + case $SIZE_LABEL in + 1KB) PAYLOAD="$P_1K" ;; + 10KB) PAYLOAD="$P_10K" ;; + 64KB) PAYLOAD="$P_64K" ;; + 256KB) PAYLOAD="$P_256K" ;; + esac + + # Записываем payload в файл, чтобы избежать проблем с экранированием + echo "$PAYLOAD" > "${TMPDIR_B}/payload_${SIZE_LABEL}" + + # Yandex + > "${TMPDIR_B}/sz_y_${SIZE_LABEL}" + for i in $(seq 1 3); do + s=$(ms_now) + ysqs send-message --queue-url "$Y_QURL" --message-body "file://${TMPDIR_B}/payload_${SIZE_LABEL}" >/dev/null 2>&1 + echo $(( $(ms_now) - s )) >> "${TMPDIR_B}/sz_y_${SIZE_LABEL}" + done + Y_SZ_AVG=$(awk '{s+=$1}END{printf "%d",s/NR}' "${TMPDIR_B}/sz_y_${SIZE_LABEL}") + + # Ours + > "${TMPDIR_B}/sz_o_${SIZE_LABEL}" + for i in $(seq 1 3); do + s=$(ms_now) + osqs send-message --queue-url "$O_QURL" --message-body "file://${TMPDIR_B}/payload_${SIZE_LABEL}" >/dev/null 2>&1 + echo $(( $(ms_now) - s )) >> "${TMPDIR_B}/sz_o_${SIZE_LABEL}" + done + O_SZ_AVG=$(awk '{s+=$1}END{printf "%d",s/NR}' "${TMPDIR_B}/sz_o_${SIZE_LABEL}") + + printf " │ %-8s │ %8d │ %8d │\n" "$SIZE_LABEL" "$Y_SZ_AVG" "$O_SZ_AVG" +done +echo " └──────────┴──────────────────┴──────────────────┘" + +ysqs purge-queue --queue-url "$Y_QURL" >/dev/null 2>&1 +osqs purge-queue --queue-url "$O_QURL" >/dev/null 2>&1 +sleep 3 + +# ═══════════════════════════════════════ +# 5. BATCH EFFICIENCY — batch vs single +# ═══════════════════════════════════════ +section "Batch efficiency — batch(10) vs 10×single" + +echo " ┌────────────────┬──────────────────┬──────────────────┐" +echo " │ Method │ Yandex avg (ms) │ Ours avg (ms) │" +echo " ├────────────────┼──────────────────┼──────────────────┤" + +# 10 × single +> "${TMPDIR_B}/be_y_single" +> "${TMPDIR_B}/be_o_single" +for i in $(seq 1 3); do + s=$(ms_now) + for j in $(seq 1 10); do ysqs send-message --queue-url "$Y_QURL" --message-body "be-s-${j}" >/dev/null 2>&1; done + echo $(( $(ms_now) - s )) >> "${TMPDIR_B}/be_y_single" + + s=$(ms_now) + for j in $(seq 1 10); do osqs send-message --queue-url "$O_QURL" --message-body "be-s-${j}" >/dev/null 2>&1; done + echo $(( $(ms_now) - s )) >> "${TMPDIR_B}/be_o_single" +done +Y_SINGLE_AVG=$(awk '{s+=$1}END{printf "%d",s/NR}' "${TMPDIR_B}/be_y_single") +O_SINGLE_AVG=$(awk '{s+=$1}END{printf "%d",s/NR}' "${TMPDIR_B}/be_o_single") +printf " │ %-14s │ %8d │ %8d │\n" "10×Single" "$Y_SINGLE_AVG" "$O_SINGLE_AVG" + +# 1 × batch(10) +> "${TMPDIR_B}/be_y_batch" +> "${TMPDIR_B}/be_o_batch" +for i in $(seq 1 3); do + s=$(ms_now) + ysqs send-message-batch --queue-url "$Y_QURL" --entries "$BATCH_ENTRIES" >/dev/null 2>&1 + echo $(( $(ms_now) - s )) >> "${TMPDIR_B}/be_y_batch" + + s=$(ms_now) + osqs send-message-batch --queue-url "$O_QURL" --entries "$BATCH_ENTRIES" >/dev/null 2>&1 + echo $(( $(ms_now) - s )) >> "${TMPDIR_B}/be_o_batch" +done +Y_BATCH_AVG=$(awk '{s+=$1}END{printf "%d",s/NR}' "${TMPDIR_B}/be_y_batch") +O_BATCH_AVG=$(awk '{s+=$1}END{printf "%d",s/NR}' "${TMPDIR_B}/be_o_batch") +printf " │ %-14s │ %8d │ %8d │\n" "Batch(10)" "$Y_BATCH_AVG" "$O_BATCH_AVG" + +Y_BATCH_GAIN=$(( (Y_SINGLE_AVG - Y_BATCH_AVG) * 100 / (Y_SINGLE_AVG + 1) )) +O_BATCH_GAIN=$(( (O_SINGLE_AVG - O_BATCH_AVG) * 100 / (O_SINGLE_AVG + 1) )) +printf " │ %-14s │ %7d%% │ %7d%% │\n" "Batch savings" "$Y_BATCH_GAIN" "$O_BATCH_GAIN" +echo " └────────────────┴──────────────────┴──────────────────┘" + +ysqs purge-queue --queue-url "$Y_QURL" >/dev/null 2>&1 +osqs purge-queue --queue-url "$O_QURL" >/dev/null 2>&1 +sleep 3 + +# ═══════════════════════════════════════ +# 6. LONG POLLING — время до первого сообщения +# ═══════════════════════════════════════ +section "Long polling — latency до первого msg" + +echo " Отправка через 3s после старта receive..." + +echo " ┌────────────────┬──────────────────┬──────────────────┐" +echo " │ WaitTimeSec │ Yandex (ms) │ Ours (ms) │" +echo " ├────────────────┼──────────────────┼──────────────────┤" + +for WTS in 5 20; do + # Yandex: запускаем receive в фоне, через 3s отправляем + Y_LP_START=$(ms_now) + ysqs receive-message --queue-url "$Y_QURL" --max-number-of-messages 1 --wait-time-seconds "$WTS" > "${TMPDIR_B}/lp_y" 2>&1 & + LP_PID=$! + sleep 3 + ysqs send-message --queue-url "$Y_QURL" --message-body "longpoll-${WTS}" >/dev/null 2>&1 + wait $LP_PID 2>/dev/null + Y_LP_MS=$(( $(ms_now) - Y_LP_START )) + Y_LP_GOT=$(grep -c "MessageId" "${TMPDIR_B}/lp_y" || true) + + # Ours + O_LP_START=$(ms_now) + osqs receive-message --queue-url "$O_QURL" --max-number-of-messages 1 --wait-time-seconds "$WTS" > "${TMPDIR_B}/lp_o" 2>&1 & + LP_PID=$! + sleep 3 + osqs send-message --queue-url "$O_QURL" --message-body "longpoll-${WTS}" >/dev/null 2>&1 + wait $LP_PID 2>/dev/null + O_LP_MS=$(( $(ms_now) - O_LP_START )) + O_LP_GOT=$(grep -c "MessageId" "${TMPDIR_B}/lp_o" || true) + + printf " │ WaitTime=%-5d │ %5dms (got:%d) │ %5dms (got:%d) │\n" \ + "$WTS" "$Y_LP_MS" "$Y_LP_GOT" "$O_LP_MS" "$O_LP_GOT" +done +echo " └────────────────┴──────────────────┴──────────────────┘" + +# Почистим оставшиеся +ysqs purge-queue --queue-url "$Y_QURL" >/dev/null 2>&1 +osqs purge-queue --queue-url "$O_QURL" >/dev/null 2>&1 +sleep 2 + +# ═══════════════════════════════════════ +# 7. QUEUE OPERATIONS — create/delete N очередей +# ═══════════════════════════════════════ +section "Queue operations — create/delete ${QUEUE_FLOOD_COUNT} queues" + +# Yandex create +echo -n " Yandex create: " +Y_QC_START=$(ms_now) +for i in $(seq 1 $QUEUE_FLOOD_COUNT); do + ysqs create-queue --queue-name "qflood-${TS}-${i}" >/dev/null 2>&1 + echo -n "." +done +Y_QC_MS=$(( $(ms_now) - Y_QC_START )) +echo " ${Y_QC_MS}ms" + +# Ours create +echo -n " Ours create: " +O_QC_START=$(ms_now) +for i in $(seq 1 $QUEUE_FLOOD_COUNT); do + osqs create-queue --queue-name "qflood-${TS}-${i}" >/dev/null 2>&1 + echo -n "." +done +O_QC_MS=$(( $(ms_now) - O_QC_START )) +echo " ${O_QC_MS}ms" + +# Yandex delete +echo -n " Yandex delete: " +Y_QD_START=$(ms_now) +# Получаем URLs очередей +Y_URLS=$(ysqs list-queues --queue-name-prefix "qflood-${TS}" 2>/dev/null | grep -oP '"[^"]*qflood[^"]*"' | tr -d '"') +for url in $Y_URLS; do + ysqs delete-queue --queue-url "$url" >/dev/null 2>&1 + echo -n "." +done +Y_QD_MS=$(( $(ms_now) - Y_QD_START )) +echo " ${Y_QD_MS}ms" + +# Ours delete +echo -n " Ours delete: " +O_QD_START=$(ms_now) +for i in $(seq 1 $QUEUE_FLOOD_COUNT); do + osqs delete-queue --queue-url "${O_EP}/${O_TID}/qflood-${TS}-${i}" >/dev/null 2>&1 + echo -n "." +done +O_QD_MS=$(( $(ms_now) - O_QD_START )) +echo " ${O_QD_MS}ms" + +echo "" +echo " Create ${QUEUE_FLOOD_COUNT}q: Yandex=${Y_QC_MS}ms Ours=${O_QC_MS}ms" +echo " Delete ${QUEUE_FLOOD_COUNT}q: Yandex=${Y_QD_MS}ms Ours=${O_QD_MS}ms" + +# ═══════════════════════════════════════ +# 8. ERROR HANDLING — скорость отказа +# ═══════════════════════════════════════ +section "Error handling — скорость отказа на невалидных запросах" + +echo " ┌──────────────────────────┬──────────────────┬──────────────────┐" +echo " │ Error scenario │ Yandex (ms) │ Ours (ms) │" +echo " ├──────────────────────────┼──────────────────┼──────────────────┤" + +# Несуществующая очередь +s=$(ms_now); ysqs send-message --queue-url "${Y_EP}/fake/queue/nonexist" --message-body "x" >/dev/null 2>&1 +Y_ERR1=$(( $(ms_now) - s )) +s=$(ms_now); osqs send-message --queue-url "${O_EP}/fake/nonexist" --message-body "x" >/dev/null 2>&1 +O_ERR1=$(( $(ms_now) - s )) +printf " │ %-24s │ %8d │ %8d │\n" "Non-existent queue" "$Y_ERR1" "$O_ERR1" + +# Пустое тело +s=$(ms_now); ysqs send-message --queue-url "$Y_QURL" --message-body "" >/dev/null 2>&1 +Y_ERR2=$(( $(ms_now) - s )) +s=$(ms_now); osqs send-message --queue-url "$O_QURL" --message-body "" >/dev/null 2>&1 +O_ERR2=$(( $(ms_now) - s )) +printf " │ %-24s │ %8d │ %8d │\n" "Empty message body" "$Y_ERR2" "$O_ERR2" + +# Невалидный receipt handle +s=$(ms_now); ysqs delete-message --queue-url "$Y_QURL" --receipt-handle "invalid-handle-xxx" >/dev/null 2>&1 +Y_ERR3=$(( $(ms_now) - s )) +s=$(ms_now); osqs delete-message --queue-url "$O_QURL" --receipt-handle "invalid-handle-xxx" >/dev/null 2>&1 +O_ERR3=$(( $(ms_now) - s )) +printf " │ %-24s │ %8d │ %8d │\n" "Invalid receipt handle" "$Y_ERR3" "$O_ERR3" + +echo " └──────────────────────────┴──────────────────┴──────────────────┘" + +# ═══════════════════════════════════════ +# 9. SUSTAINED LOAD — непрерывная нагрузка 2 мин +# ═══════════════════════════════════════ +section "Sustained load — ${SUSTAINED_DURATION}s непрерывно (send+receive+delete)" + +run_sustained() { + local provider="$1" sqs_func="$2" qurl="$3" duration="$4" result_file="$5" + local end=$((SECONDS + duration)) + local sent=0 recv=0 del=0 errors=0 + local interval=30 + local next_report=$((SECONDS + interval)) + + while [[ $SECONDS -lt $end ]]; do + # Send + R=$($sqs_func send-message --queue-url "$qurl" --message-body "sustained-$(date +%s%N)" 2>&1) + if echo "$R" | grep -q "MessageId"; then + sent=$((sent+1)) + else + errors=$((errors+1)) + fi + + # Receive + Delete + R=$($sqs_func receive-message --queue-url "$qurl" --max-number-of-messages 1 --wait-time-seconds 0 2>&1) + if echo "$R" | grep -q "MessageId"; then + recv=$((recv+1)) + H=$(echo "$R" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+' | head -1) + if [[ -n "$H" ]]; then + $sqs_func delete-message --queue-url "$qurl" --receipt-handle "$H" >/dev/null 2>&1 + del=$((del+1)) + fi + fi + + # Промежуточный отчёт + if [[ $SECONDS -ge $next_report ]]; then + local elapsed=$((SECONDS - (end - duration))) + echo " [${provider} ${elapsed}s] sent=$sent recv=$recv del=$del err=$errors" + next_report=$((SECONDS + interval)) + fi + done + echo "${sent}:${recv}:${del}:${errors}" > "$result_file" +} + +echo " Запуск Yandex sustained..." +run_sustained "YMQ" "ysqs" "$Y_QURL" "$SUSTAINED_DURATION" "${TMPDIR_B}/sus_y" & +Y_SUS_PID=$! + +echo " Запуск Ours sustained..." +run_sustained "OURS" "osqs" "$O_QURL" "$SUSTAINED_DURATION" "${TMPDIR_B}/sus_o" & +O_SUS_PID=$! + +echo " ⏳ Ждём ${SUSTAINED_DURATION}s..." +wait $Y_SUS_PID $O_SUS_PID 2>/dev/null + +IFS=':' read -r Y_SUS_SENT Y_SUS_RECV Y_SUS_DEL Y_SUS_ERR < "${TMPDIR_B}/sus_y" +IFS=':' read -r O_SUS_SENT O_SUS_RECV O_SUS_DEL O_SUS_ERR < "${TMPDIR_B}/sus_o" + +Y_SUS_RPS=$(( (Y_SUS_SENT + Y_SUS_RECV + Y_SUS_DEL) / (SUSTAINED_DURATION + 1) )) +O_SUS_RPS=$(( (O_SUS_SENT + O_SUS_RECV + O_SUS_DEL) / (SUSTAINED_DURATION + 1) )) + +echo "" +echo " ┌──────────────┬─────────────────────────────────────┬─────────────────────────────────────┐" +echo " │ │ Yandex MQ │ shared-sqs │" +echo " ├──────────────┼─────────────────────────────────────┼─────────────────────────────────────┤" +printf " │ %-12s │ sent=%-4d recv=%-4d del=%-4d err=%-3d │ sent=%-4d recv=%-4d del=%-4d err=%-3d │\n" \ + "Operations" "$Y_SUS_SENT" "$Y_SUS_RECV" "$Y_SUS_DEL" "$Y_SUS_ERR" \ + "$O_SUS_SENT" "$O_SUS_RECV" "$O_SUS_DEL" "$O_SUS_ERR" +printf " │ %-12s │ ~%-3d ops/sec │ ~%-3d ops/sec │\n" \ + "Throughput" "$Y_SUS_RPS" "$O_SUS_RPS" +echo " └──────────────┴─────────────────────────────────────┴─────────────────────────────────────┘" + +# ═══════════════════════════════════════ +# 10. CLEANUP +# ═══════════════════════════════════════ +section "Cleanup" +ysqs purge-queue --queue-url "$Y_QURL" >/dev/null 2>&1 +ysqs delete-queue --queue-url "$Y_QURL" >/dev/null 2>&1 +echo " Yandex: bench queue deleted" +osqs purge-queue --queue-url "$O_QURL" >/dev/null 2>&1 +osqs delete-queue --queue-url "$O_QURL" >/dev/null 2>&1 +echo " Ours: bench queue deleted" + +# ═══════════════════════════════════════ +# ИТОГОВАЯ СВОДКА +# ═══════════════════════════════════════ +echo "" +echo "╔══════════════════════════════════════════════════════════════════════╗" +echo "║ FULL BENCHMARK COMPLETE ║" +echo "║ $(date '+%Y-%m-%d %H:%M:%S') ║" +echo "║ ║" +echo "║ Latency: SendMessage avg Y=$(echo $Y_SEND_STATS | awk '{print $2}')ms O=$(echo $O_SEND_STATS | awk '{print $2}')ms" +echo "║ ReceiveMsg avg Y=$(echo $Y_RECV_STATS | awk '{print $2}')ms O=$(echo $O_RECV_STATS | awk '{print $2}')ms" +echo "║ Batch: savings Y=${Y_BATCH_GAIN}% O=${O_BATCH_GAIN}%" +echo "║ Sustained: ops/sec Y=${Y_SUS_RPS} O=${O_SUS_RPS} (${SUSTAINED_DURATION}s)" +echo "║ Errors: Y=${Y_SUS_ERR} O=${O_SUS_ERR}" +echo "║ QueueOps: create10 Y=${Y_QC_MS}ms O=${O_QC_MS}ms" +echo "║ delete10 Y=${Y_QD_MS}ms O=${O_QD_MS}ms" +echo "║ Tags: shared-sqs ✅ (3/3) Yandex ⚠️ not supported" +echo "╚══════════════════════════════════════════════════════════════════════╝" diff --git a/tests/stress_test_v2.sh b/tests/stress_test_v2.sh new file mode 100644 index 0000000..bb4d139 --- /dev/null +++ b/tests/stress_test_v2.sh @@ -0,0 +1,966 @@ +#!/bin/bash +# stress_test.sh — ЖЁСТКИЙ стресс-тест shared-sqs +# Created: 2026-04-11 +# Updated: 2026-04-11 — v2: масштаб, long-poll, race, batch, 50 очередей, memory, multi-kill +# Покрывает: +# 1. Подготовка (тенанты, очереди) +# 2. Параллельная отправка — 50 воркеров × 20 msg = 1000 сообщений +# 3. Параллельное чтение + удаление (гонка) +# 4. Multi-tenant изоляция (5 тенантов одновременно) +# 5. Burst — 100 сообщений одновременно +# 6. Long-polling + параллельные producer'ы +# 7. Двойное удаление (race на receipt handle) +# 8. Batch-операции конкурентно (SendMessageBatch + DeleteMessageBatch) +# 9. Много очередей — 30 очередей на тенанта +# 10. Kill pod + восстановление из Redis +# 11. Redis disconnect / reconnect +# 12. Смешанная нагрузка (send+recv+delete+getattr, 20s) +# 13. Memory leak check (RSS до/после нагрузки) +# 14. Multi-kill: kill → восстановление → kill → восстановление +# 15. Cleanup +# Запуск: BASE_URL=https://qu.kube5s.ru ADMIN_TOKEN=... bash tests/stress_test.sh + +set -uo pipefail + +# ═══════════════════════════════════════════ +# КОНФИГУРАЦИЯ +# ═══════════════════════════════════════════ +BASE_URL="${BASE_URL:-https://qu.kube5s.ru}" +ADMIN_TOKEN="${ADMIN_TOKEN:-sqs-admin-7a7d8bd0c060a75c198d48680f34077a}" +TS=$(date +%s) +NAMESPACE="shared-sqs" +POD_LABEL="app.kubernetes.io/name=shared-sqs" + +# Параметры нагрузки — ЖЁСТКИЕ +CONCURRENT_WORKERS=50 # параллельных воркеров +MESSAGES_PER_WORKER=20 # сообщений на воркер (итого 1000) +BURST_SIZE=100 # размер burst-пакета +TENANT_COUNT=5 # тенантов для multi-tenant теста +QUEUE_FLOOD_COUNT=30 # очередей на тенанта +MIXED_DURATION=20 # длительность смешанной нагрузки (сек) +BATCH_WORKERS=10 # воркеров для batch-операций + +PASS=0 +FAIL=0 +TOTAL=0 +TMPDIR_STRESS=$(mktemp -d /tmp/sqs_stress_XXXXX) + +# ═══════════════════════════════════════════ +# ХЕЛПЕРЫ +# ═══════════════════════════════════════════ + +json_field() { + python3 -c "import sys,json; print(json.load(sys.stdin).get('$1',''))" 2>/dev/null +} + +# progress_dot — печатает точку без переноса строки (прогресс) +progress_dot() { + printf "." >&2 +} + +# progress_done — завершает строку прогресса +progress_done() { + echo "" >&2 +} + +check() { + local name="$1" output="$2" pattern="$3" + TOTAL=$((TOTAL + 1)) + if echo "$output" | grep -qE "$pattern"; then + PASS=$((PASS + 1)) + echo " ✅ $name" + else + FAIL=$((FAIL + 1)) + echo " ❌ $name" + echo " ожидалось: $pattern" + echo " получено: $(echo "$output" | head -5)" + fi +} + +check_eq() { + local name="$1" got="$2" want="$3" + TOTAL=$((TOTAL + 1)) + if [[ "$got" == "$want" ]]; then + PASS=$((PASS + 1)) + echo " ✅ $name ($got)" + else + FAIL=$((FAIL + 1)) + echo " ❌ $name: ожидалось=$want, получено=$got" + fi +} + +check_ge() { + local name="$1" got="$2" want="$3" + TOTAL=$((TOTAL + 1)) + if [[ "$got" -ge "$want" ]]; then + PASS=$((PASS + 1)) + echo " ✅ $name ($got >= $want)" + else + FAIL=$((FAIL + 1)) + echo " ❌ $name: ожидалось >=$want, получено=$got" + fi +} + +check_le() { + local name="$1" got="$2" want="$3" + TOTAL=$((TOTAL + 1)) + if [[ "$got" -le "$want" ]]; then + PASS=$((PASS + 1)) + echo " ✅ $name ($got <= $want)" + else + FAIL=$((FAIL + 1)) + echo " ❌ $name: ожидалось <=$want, получено=$got" + fi +} + +sqs() { + local ak="$1" sk="$2" + shift 2 + AWS_ACCESS_KEY_ID="$ak" \ + AWS_SECRET_ACCESS_KEY="$sk" \ + AWS_DEFAULT_REGION="us-east-1" \ + aws --endpoint-url "$BASE_URL" --output json sqs "$@" 2>&1 +} + +admin_api() { + local method="$1" path="$2" body="${3:-}" + if [[ -n "$body" ]]; then + curl -s --max-time 20 -X "$method" \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d "$body" "${BASE_URL}${path}" 2>&1 + else + curl -s --max-time 20 -X "$method" \ + -H "Authorization: Bearer $ADMIN_TOKEN" "${BASE_URL}${path}" 2>&1 + fi +} + +qurl() { echo "${BASE_URL}/${1}/${2}"; } + +wait_for_pod_ready() { + local max_wait=120 waited=0 + echo " ⏳ Ожидание пода (max ${max_wait}s)..." + while [[ $waited -lt $max_wait ]]; do + local ready + ready=$(kubectl get pods -n "$NAMESPACE" -l "$POD_LABEL" \ + -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null) + if [[ "$ready" == "True" ]]; then + echo " ✅ Под готов (${waited}s)" + return 0 + fi + sleep 2 + waited=$((waited + 2)) + printf "." >&2 + done + echo "" + echo " ⚠️ Под не готов после ${max_wait}s" + return 1 +} + +wait_for_healthy() { + local max_wait=120 waited=0 + echo " ⏳ Ожидание /health (max ${max_wait}s)..." + while [[ $waited -lt $max_wait ]]; do + local status + status=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "${BASE_URL}/health" 2>/dev/null) + if [[ "$status" == "200" ]]; then + echo " ✅ Сервис отвечает (${waited}s)" + return 0 + fi + sleep 2 + waited=$((waited + 2)) + printf "." >&2 + done + echo "" + echo " ⚠️ Сервис не отвечает после ${max_wait}s" + return 1 +} + +# get_pod_rss_mb — RSS памяти текущего пода в MB +get_pod_rss_mb() { + local pod + pod=$(kubectl get pods -n "$NAMESPACE" -l "$POD_LABEL" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + # kubectl top может не работать — fallback на /proc + local rss_kb + rss_kb=$(kubectl exec -n "$NAMESPACE" "$pod" -- cat /proc/1/status 2>/dev/null \ + | grep VmRSS | awk '{print $2}') + if [[ -n "$rss_kb" ]]; then + echo $((rss_kb / 1024)) + else + echo "0" + fi +} + +kill_pod() { + local pod + pod=$(kubectl get pods -n "$NAMESPACE" -l "$POD_LABEL" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + echo " 🔪 Kill: $pod" + kubectl delete pod "$pod" -n "$NAMESPACE" --grace-period=0 --force 2>&1 | head -1 +} + +cleanup() { rm -rf "$TMPDIR_STRESS" 2>/dev/null; } +trap cleanup EXIT + +STARTED_AT=$(date +%s) + +echo "╔═══════════════════════════════════════════════════════════════════╗" +echo "║ shared-sqs STRESS TEST v2 — $(date '+%Y-%m-%d %H:%M:%S') ║" +echo "║ Endpoint: ${BASE_URL} ║" +echo "║ Workers: ${CONCURRENT_WORKERS} Msgs/worker: ${MESSAGES_PER_WORKER} Burst: ${BURST_SIZE} Tenants: ${TENANT_COUNT} ║" +echo "║ Queue-flood: ${QUEUE_FLOOD_COUNT} Batch-workers: ${BATCH_WORKERS} Mixed: ${MIXED_DURATION}s ║" +echo "╚═══════════════════════════════════════════════════════════════════╝" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 1. ПОДГОТОВКА — тенанты и очереди +# ═══════════════════════════════════════════════════════════ +echo "═══ 1. Подготовка — создаём $TENANT_COUNT тенантов ═══" + +declare -a TENANT_AK TENANT_SK TENANT_ID +for i in $(seq 1 $TENANT_COUNT); do + R=$(admin_api POST "/admin/tenants" "{\"name\":\"stress-t${i}-${TS}\",\"max_queues\":100}") + TENANT_AK[$i]=$(echo "$R" | json_field access_key) + TENANT_SK[$i]=$(echo "$R" | json_field secret_key) + TENANT_ID[$i]=$(echo "$R" | json_field id) + echo " Тенант ${i}: ID=${TENANT_ID[$i]} AK=${TENANT_AK[$i]:0:20}..." + if [[ -z "${TENANT_AK[$i]}" ]]; then + echo " ⛔ ОШИБКА создания тенанта ${i}: $R" + exit 1 + fi +done + +# Сохраняем в файлы — для subshell +for i in $(seq 1 $TENANT_COUNT); do + echo "${TENANT_AK[$i]}" > "${TMPDIR_STRESS}/tak_${i}" + echo "${TENANT_SK[$i]}" > "${TMPDIR_STRESS}/tsk_${i}" + echo "${TENANT_ID[$i]}" > "${TMPDIR_STRESS}/tid_${i}" +done + +T1_AK="${TENANT_AK[1]}"; T1_SK="${TENANT_SK[1]}"; T1_ID="${TENANT_ID[1]}" + +# Очереди +STRESS_Q="stress-conc-${TS}" +for i in $(seq 1 $TENANT_COUNT); do + R=$(sqs "${TENANT_AK[$i]}" "${TENANT_SK[$i]}" create-queue --queue-name "$STRESS_Q") +done +STRESS_QURL=$(qurl "$T1_ID" "$STRESS_Q") + +BURST_Q="stress-burst-${TS}" +R=$(sqs "$T1_AK" "$T1_SK" create-queue --queue-name "$BURST_Q") +BURST_QURL=$(qurl "$T1_ID" "$BURST_Q") + +echo " Очередь conc: $STRESS_Q (для $TENANT_COUNT тенантов)" +echo " Очередь burst: $BURST_Q" +echo " Подготовка завершена за $(($(date +%s) - STARTED_AT))s" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 2. КОНКУРЕНТНАЯ ОТПРАВКА — 50 × 20 = 1000 сообщений +# ═══════════════════════════════════════════════════════════ +TOTAL_MSGS=$((CONCURRENT_WORKERS * MESSAGES_PER_WORKER)) +echo "═══ 2. Конкурентная отправка: ${CONCURRENT_WORKERS} воркеров × ${MESSAGES_PER_WORKER} = ${TOTAL_MSGS} сообщений ═══" +SEC2_START=$(date +%s) + +for w in $(seq 1 $CONCURRENT_WORKERS); do + ( + ok=0; fail=0 + for m in $(seq 1 $MESSAGES_PER_WORKER); do + R=$(sqs "$T1_AK" "$T1_SK" send-message \ + --queue-url "$STRESS_QURL" \ + --message-body "w${w}-m${m}-${TS}" 2>&1) + if echo "$R" | grep -q "MessageId"; then ok=$((ok + 1)); else fail=$((fail + 1)); fi + done + echo "${ok}:${fail}" > "${TMPDIR_STRESS}/send_w${w}.result" + # Прогресс: каждый 10-й воркер печатает + if (( w % 10 == 0 )); then printf "[W${w}:${ok}ok] " >&2; fi + ) & +done +echo " ⏳ Отправка идёт... (прогресс каждые 10 воркеров)" +wait +echo "" + +SEND_OK=0; SEND_FAIL=0 +for w in $(seq 1 $CONCURRENT_WORKERS); do + if [[ -f "${TMPDIR_STRESS}/send_w${w}.result" ]]; then + IFS=':' read -r ok fail < "${TMPDIR_STRESS}/send_w${w}.result" + SEND_OK=$((SEND_OK + ok)); SEND_FAIL=$((SEND_FAIL + fail)) + fi +done +SEC2_DUR=$(($(date +%s) - SEC2_START)) +echo " Результат: ${SEND_OK} ok, ${SEND_FAIL} fail за ${SEC2_DUR}s (~$((SEND_OK / (SEC2_DUR + 1))) msg/s)" +check_eq "Все ${TOTAL_MSGS} сообщений отправлены" "$SEND_OK" "$TOTAL_MSGS" + +sleep 2 +R=$(sqs "$T1_AK" "$T1_SK" get-queue-attributes \ + --queue-url "$STRESS_QURL" --attribute-names ApproximateNumberOfMessages 2>&1) +APPROX=$(echo "$R" | grep -oP '"ApproximateNumberOfMessages"\s*:\s*"\K[0-9]+') +echo " GetQueueAttributes: ~${APPROX} в очереди" +check_eq "Количество в очереди = $TOTAL_MSGS" "${APPROX:-0}" "$TOTAL_MSGS" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 3. КОНКУРЕНТНОЕ ЧТЕНИЕ — гонка за 1000 сообщений +# ═══════════════════════════════════════════════════════════ +echo "═══ 3. Конкурентное чтение: ${CONCURRENT_WORKERS} воркеров забирают ${TOTAL_MSGS} сообщений ═══" +SEC3_START=$(date +%s) + +for w in $(seq 1 $CONCURRENT_WORKERS); do + ( + received=0; deleted=0; empty_rounds=0 + while [[ $empty_rounds -lt 8 ]]; do + R=$(sqs "$T1_AK" "$T1_SK" receive-message \ + --queue-url "$STRESS_QURL" \ + --max-number-of-messages 10 --wait-time-seconds 1 2>&1) + MSG_COUNT=$(echo "$R" | grep -c '"MessageId"' || true) + if [[ "$MSG_COUNT" -eq 0 ]]; then empty_rounds=$((empty_rounds + 1)); continue; fi + empty_rounds=0 + received=$((received + MSG_COUNT)) + HANDLES=$(echo "$R" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+') + while IFS= read -r handle; do + sqs "$T1_AK" "$T1_SK" delete-message \ + --queue-url "$STRESS_QURL" --receipt-handle "$handle" >/dev/null 2>&1 + deleted=$((deleted + 1)) + done <<< "$HANDLES" + done + echo "${received}:${deleted}" > "${TMPDIR_STRESS}/recv_w${w}.result" + if (( w % 10 == 0 )); then printf "[R${w}:${received}r/${deleted}d] " >&2; fi + ) & +done +echo " ⏳ Чтение идёт... (прогресс каждые 10 воркеров)" +wait +echo "" + +RECV_TOTAL=0; DEL_TOTAL=0 +for w in $(seq 1 $CONCURRENT_WORKERS); do + if [[ -f "${TMPDIR_STRESS}/recv_w${w}.result" ]]; then + IFS=':' read -r recv del < "${TMPDIR_STRESS}/recv_w${w}.result" + RECV_TOTAL=$((RECV_TOTAL + recv)); DEL_TOTAL=$((DEL_TOTAL + del)) + fi +done +SEC3_DUR=$(($(date +%s) - SEC3_START)) +echo " Результат: прочитано=${RECV_TOTAL}, удалено=${DEL_TOTAL} за ${SEC3_DUR}s" +check_ge "Прочитано >= $TOTAL_MSGS" "$RECV_TOTAL" "$TOTAL_MSGS" + +sleep 2 +R=$(sqs "$T1_AK" "$T1_SK" get-queue-attributes \ + --queue-url "$STRESS_QURL" --attribute-names ApproximateNumberOfMessages 2>&1) +APPROX=$(echo "$R" | grep -oP '"ApproximateNumberOfMessages"\s*:\s*"\K[0-9]+') +check_eq "Очередь пуста" "${APPROX:-0}" "0" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 4. MULTI-TENANT ИЗОЛЯЦИЯ — 5 тенантов × 50 msg одновременно +# ═══════════════════════════════════════════════════════════ +MSGS_PER_TENANT=50 +echo "═══ 4. Multi-tenant изоляция: ${TENANT_COUNT} тенантов × ${MSGS_PER_TENANT} msg ═══" +SEC4_START=$(date +%s) + +for i in $(seq 1 $TENANT_COUNT); do + ( + tak=$(cat "${TMPDIR_STRESS}/tak_${i}") + tsk=$(cat "${TMPDIR_STRESS}/tsk_${i}") + tid=$(cat "${TMPDIR_STRESS}/tid_${i}") + my_qurl="${BASE_URL}/${tid}/${STRESS_Q}" + ok=0 + for m in $(seq 1 $MSGS_PER_TENANT); do + R=$(sqs "$tak" "$tsk" send-message --queue-url "$my_qurl" \ + --message-body "tenant-${i}-iso-${m}" 2>&1) + if echo "$R" | grep -q "MessageId"; then ok=$((ok + 1)); fi + done + echo "$ok" > "${TMPDIR_STRESS}/tenant_send_${i}.result" + printf "[T${i}:${ok}sent] " >&2 + ) & +done +echo " ⏳ Тенанты отправляют..." +wait +echo "" + +# Чтение и проверка изоляции +for i in $(seq 1 $TENANT_COUNT); do + ( + tak=$(cat "${TMPDIR_STRESS}/tak_${i}") + tsk=$(cat "${TMPDIR_STRESS}/tsk_${i}") + tid=$(cat "${TMPDIR_STRESS}/tid_${i}") + my_qurl="${BASE_URL}/${tid}/${STRESS_Q}" + own=0; foreign=0; empty_rounds=0 + while [[ $empty_rounds -lt 5 ]]; do + R=$(sqs "$tak" "$tsk" receive-message --queue-url "$my_qurl" \ + --max-number-of-messages 10 --wait-time-seconds 1 2>&1) + MSG_COUNT=$(echo "$R" | grep -c '"Body"' || true) + if [[ "$MSG_COUNT" -eq 0 ]]; then empty_rounds=$((empty_rounds + 1)); continue; fi + empty_rounds=0 + while IFS= read -r body; do + if echo "$body" | grep -q "tenant-${i}-"; then own=$((own + 1)); else foreign=$((foreign + 1)); fi + done < <(echo "$R" | grep -oP '"Body"\s*:\s*"\K[^"]+') + HANDLES=$(echo "$R" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+') + while IFS= read -r handle; do + sqs "$tak" "$tsk" delete-message --queue-url "$my_qurl" --receipt-handle "$handle" >/dev/null 2>&1 + done <<< "$HANDLES" + done + echo "${own}:${foreign}" > "${TMPDIR_STRESS}/tenant_recv_${i}.result" + printf "[T${i}:own=${own},foreign=${foreign}] " >&2 + ) & +done +echo " ⏳ Тенанты читают..." +wait +echo "" + +ALL_FOREIGN=0; ALL_OWN=0 +for i in $(seq 1 $TENANT_COUNT); do + if [[ -f "${TMPDIR_STRESS}/tenant_recv_${i}.result" ]]; then + IFS=':' read -r own foreign < "${TMPDIR_STRESS}/tenant_recv_${i}.result" + echo " Тенант ${i}: своих=${own}, чужих=${foreign}" + ALL_OWN=$((ALL_OWN + own)); ALL_FOREIGN=$((ALL_FOREIGN + foreign)) + fi +done +SEC4_DUR=$(($(date +%s) - SEC4_START)) +echo " Итого: своих=$ALL_OWN, чужих=$ALL_FOREIGN за ${SEC4_DUR}s" +check_eq "Чужих сообщений: 0" "$ALL_FOREIGN" "0" +check_ge "Свои сообщения получены" "$ALL_OWN" "$((TENANT_COUNT * MSGS_PER_TENANT))" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 5. BURST — 100 сообщений одновременно +# ═══════════════════════════════════════════════════════════ +echo "═══ 5. Burst: ${BURST_SIZE} сообщений одновременно ═══" +SEC5_START=$(date +%s) + +for b in $(seq 1 $BURST_SIZE); do + ( + R=$(sqs "$T1_AK" "$T1_SK" send-message --queue-url "$BURST_QURL" \ + --message-body "burst-${b}-${TS}" 2>&1) + if echo "$R" | grep -q "MessageId"; then echo "1"; else echo "0"; fi \ + > "${TMPDIR_STRESS}/burst_${b}.result" + ) & + # Прогресс пачками по 25 + if (( b % 25 == 0 )); then printf "[${b}/${BURST_SIZE}] " >&2; fi +done +echo " ⏳ Burst отправка..." +wait +echo "" + +BURST_OK=0 +for b in $(seq 1 $BURST_SIZE); do + [[ -f "${TMPDIR_STRESS}/burst_${b}.result" ]] && BURST_OK=$((BURST_OK + $(cat "${TMPDIR_STRESS}/burst_${b}.result"))) +done +SEC5_DUR=$(($(date +%s) - SEC5_START)) +echo " Результат: ${BURST_OK}/${BURST_SIZE} за ${SEC5_DUR}s" +check_ge "Burst >= 90%" "$BURST_OK" "$((BURST_SIZE * 90 / 100))" + +sleep 3 +R=$(sqs "$T1_AK" "$T1_SK" get-queue-attributes --queue-url "$BURST_QURL" \ + --attribute-names ApproximateNumberOfMessages 2>&1) +BURST_APPROX=$(echo "$R" | grep -oP '"ApproximateNumberOfMessages"\s*:\s*"\K[0-9]+') +echo " Очередь burst: ~${BURST_APPROX}" +check_eq "Burst все в очереди" "${BURST_APPROX:-0}" "$BURST_OK" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 6. LONG-POLLING + параллельные producer'ы +# ═══════════════════════════════════════════════════════════ +echo "═══ 6. Long-polling: consumer WaitTime=10s + 5 producers ═══" +SEC6_START=$(date +%s) + +LONGPOLL_Q="stress-longpoll-${TS}" +R=$(sqs "$T1_AK" "$T1_SK" create-queue --queue-name "$LONGPOLL_Q" \ + --attributes '{"ReceiveMessageWaitTimeSeconds":"10"}') +LONGPOLL_QURL=$(qurl "$T1_ID" "$LONGPOLL_Q") + +# Consumer: читает с long-poll, считает сколько получит за 30s +( + end=$((SECONDS + 30)) + total_recv=0 + while [[ $SECONDS -lt $end ]]; do + R=$(sqs "$T1_AK" "$T1_SK" receive-message --queue-url "$LONGPOLL_QURL" \ + --max-number-of-messages 10 --wait-time-seconds 10 2>&1) + cnt=$(echo "$R" | grep -c '"MessageId"' || true) + total_recv=$((total_recv + cnt)) + if (( total_recv % 10 == 0 && total_recv > 0 )); then printf "[LP-recv:${total_recv}] " >&2; fi + done + echo "$total_recv" > "${TMPDIR_STRESS}/longpoll_recv.result" +) & + +# Producers: начинают через 3s, каждый отправляет 20 msg с паузами +sleep 3 +for p in $(seq 1 5); do + ( + ok=0 + for m in $(seq 1 20); do + R=$(sqs "$T1_AK" "$T1_SK" send-message --queue-url "$LONGPOLL_QURL" \ + --message-body "longpoll-p${p}-m${m}" 2>&1) + if echo "$R" | grep -q "MessageId"; then ok=$((ok + 1)); fi + sleep 0.2 + done + echo "$ok" > "${TMPDIR_STRESS}/longpoll_prod_${p}.result" + printf "[LP-prod${p}:${ok}] " >&2 + ) & +done +echo " ⏳ Long-poll: consumer ждёт, producers отправляют (30s)..." +wait +echo "" + +LP_SENT=0 +for p in $(seq 1 5); do + [[ -f "${TMPDIR_STRESS}/longpoll_prod_${p}.result" ]] && LP_SENT=$((LP_SENT + $(cat "${TMPDIR_STRESS}/longpoll_prod_${p}.result"))) +done +LP_RECV=$(cat "${TMPDIR_STRESS}/longpoll_recv.result" 2>/dev/null || echo 0) +SEC6_DUR=$(($(date +%s) - SEC6_START)) +echo " Producers отправили: $LP_SENT, consumer получил: $LP_RECV за ${SEC6_DUR}s" +check_ge "Long-poll: получено >= 50% отправленных" "$LP_RECV" "$((LP_SENT / 2))" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 7. ДВОЙНОЕ УДАЛЕНИЕ — race на receipt handle +# ═══════════════════════════════════════════════════════════ +echo "═══ 7. Двойное удаление: 2 воркера гонятся за одно сообщение ═══" + +RACE_Q="stress-race-${TS}" +R=$(sqs "$T1_AK" "$T1_SK" create-queue --queue-name "$RACE_Q") +RACE_QURL=$(qurl "$T1_ID" "$RACE_Q") + +# Отправляем 20 сообщений +for m in $(seq 1 20); do + sqs "$T1_AK" "$T1_SK" send-message --queue-url "$RACE_QURL" \ + --message-body "race-${m}" >/dev/null 2>&1 +done +echo " Отправлено 20 сообщений" + +# VisibilityTimeout=0 — чтобы оба воркера получали одни сообщения +RACE_DOUBLE_DEL=0 +RACE_OK_DEL=0 +for round in $(seq 1 10); do + # Два воркера одновременно получают + R1=$(sqs "$T1_AK" "$T1_SK" receive-message --queue-url "$RACE_QURL" \ + --max-number-of-messages 1 --visibility-timeout 30 2>&1) + HANDLE=$(echo "$R1" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+' | head -1) + if [[ -z "$HANDLE" ]]; then continue; fi + + # Два параллельных delete + D1=$(sqs "$T1_AK" "$T1_SK" delete-message --queue-url "$RACE_QURL" --receipt-handle "$HANDLE" 2>&1) & + PID1=$! + D2=$(sqs "$T1_AK" "$T1_SK" delete-message --queue-url "$RACE_QURL" --receipt-handle "$HANDLE" 2>&1) & + PID2=$! + wait $PID1 $PID2 + RACE_OK_DEL=$((RACE_OK_DEL + 1)) + printf "[race-${round}] " >&2 +done +echo "" +echo " Раундов race: ${RACE_OK_DEL}/10" +# Главное: сервер НЕ упал +R=$(sqs "$T1_AK" "$T1_SK" get-queue-attributes --queue-url "$RACE_QURL" \ + --attribute-names ApproximateNumberOfMessages 2>&1) +check "Сервер жив после double-delete" "$R" "ApproximateNumberOfMessages" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 8. BATCH-ОПЕРАЦИИ конкурентно +# ═══════════════════════════════════════════════════════════ +echo "═══ 8. Batch-операции: ${BATCH_WORKERS} воркеров SendMessageBatch + DeleteMessageBatch ═══" +SEC8_START=$(date +%s) + +BATCH_Q="stress-batch-${TS}" +R=$(sqs "$T1_AK" "$T1_SK" create-queue --queue-name "$BATCH_Q") +BATCH_QURL=$(qurl "$T1_ID" "$BATCH_Q") + +# Параллельные SendMessageBatch (10 msg per batch × 10 workers = 100 msg) +for w in $(seq 1 $BATCH_WORKERS); do + ( + # Формируем batch из 10 записей + ENTRIES="" + for e in $(seq 1 10); do + [[ -n "$ENTRIES" ]] && ENTRIES="${ENTRIES}," + ENTRIES="${ENTRIES}{\"Id\":\"${w}-${e}\",\"MessageBody\":\"batch-w${w}-e${e}\"}" + done + R=$(sqs "$T1_AK" "$T1_SK" send-message-batch --queue-url "$BATCH_QURL" \ + --entries "[${ENTRIES}]" 2>&1) + ok=$(echo "$R" | grep -c '"MessageId"' || true) + echo "$ok" > "${TMPDIR_STRESS}/batch_send_${w}.result" + printf "[B-send${w}:${ok}] " >&2 + ) & +done +echo " ⏳ SendMessageBatch..." +wait +echo "" + +BATCH_SENT=0 +for w in $(seq 1 $BATCH_WORKERS); do + [[ -f "${TMPDIR_STRESS}/batch_send_${w}.result" ]] && BATCH_SENT=$((BATCH_SENT + $(cat "${TMPDIR_STRESS}/batch_send_${w}.result"))) +done +echo " Batch отправлено: ${BATCH_SENT}" +check_ge "Batch: отправлено >= 80" "$BATCH_SENT" "80" + +# Параллельные ReceiveMessage + DeleteMessageBatch +BATCH_DELETED=0 +for w in $(seq 1 5); do + ( + del=0; empty=0 + while [[ $empty -lt 3 ]]; do + R=$(sqs "$T1_AK" "$T1_SK" receive-message --queue-url "$BATCH_QURL" \ + --max-number-of-messages 10 --wait-time-seconds 1 2>&1) + HANDLES=$(echo "$R" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+') + if [[ -z "$HANDLES" ]]; then empty=$((empty + 1)); continue; fi + empty=0 + # Формируем batch delete + ENTRIES="" + idx=0 + while IFS= read -r handle; do + [[ -z "$handle" ]] && continue + [[ -n "$ENTRIES" ]] && ENTRIES="${ENTRIES}," + ENTRIES="${ENTRIES}{\"Id\":\"${idx}\",\"ReceiptHandle\":\"${handle}\"}" + idx=$((idx + 1)) + done <<< "$HANDLES" + if [[ -n "$ENTRIES" ]]; then + DR=$(sqs "$T1_AK" "$T1_SK" delete-message-batch --queue-url "$BATCH_QURL" \ + --entries "[${ENTRIES}]" 2>&1) + ok_del=$(echo "$DR" | grep -c '"Id"' || true) + del=$((del + ok_del)) + fi + done + echo "$del" > "${TMPDIR_STRESS}/batch_del_${w}.result" + printf "[B-del${w}:${del}] " >&2 + ) & +done +echo " ⏳ DeleteMessageBatch..." +wait +echo "" + +BATCH_DEL_TOTAL=0 +for w in $(seq 1 5); do + [[ -f "${TMPDIR_STRESS}/batch_del_${w}.result" ]] && BATCH_DEL_TOTAL=$((BATCH_DEL_TOTAL + $(cat "${TMPDIR_STRESS}/batch_del_${w}.result"))) +done +SEC8_DUR=$(($(date +%s) - SEC8_START)) +echo " Batch удалено: $BATCH_DEL_TOTAL за ${SEC8_DUR}s" +check_ge "Batch: удалено >= 80%" "$BATCH_DEL_TOTAL" "$((BATCH_SENT * 80 / 100))" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 9. МНОГО ОЧЕРЕДЕЙ — 30 очередей на тенанта +# ═══════════════════════════════════════════════════════════ +echo "═══ 9. Queue-flood: ${QUEUE_FLOOD_COUNT} очередей, send+receive в каждую ═══" +SEC9_START=$(date +%s) + +# Создаём 30 очередей параллельно +for q in $(seq 1 $QUEUE_FLOOD_COUNT); do + ( + qname="stress-flood-${q}-${TS}" + my_qurl=$(qurl "$T1_ID" "$qname") + # Создаём + R=$(sqs "$T1_AK" "$T1_SK" create-queue --queue-name "$qname" 2>&1) + if ! echo "$R" | grep -q "QueueUrl"; then echo "0" > "${TMPDIR_STRESS}/flood_${q}.result"; exit 0; fi + # Отправляем 5 msg + for m in $(seq 1 5); do + sqs "$T1_AK" "$T1_SK" send-message --queue-url "$my_qurl" \ + --message-body "flood-q${q}-m${m}" >/dev/null 2>&1 + done + # Читаем 1 + R=$(sqs "$T1_AK" "$T1_SK" receive-message --queue-url "$my_qurl" \ + --max-number-of-messages 1 --wait-time-seconds 1 2>&1) + if echo "$R" | grep -q "MessageId"; then echo "1" > "${TMPDIR_STRESS}/flood_${q}.result" + else echo "0" > "${TMPDIR_STRESS}/flood_${q}.result"; fi + if (( q % 10 == 0 )); then printf "[Q${q}] " >&2; fi + ) & +done +echo " ⏳ Создание и тест ${QUEUE_FLOOD_COUNT} очередей..." +wait +echo "" + +FLOOD_OK=0 +for q in $(seq 1 $QUEUE_FLOOD_COUNT); do + [[ -f "${TMPDIR_STRESS}/flood_${q}.result" ]] && FLOOD_OK=$((FLOOD_OK + $(cat "${TMPDIR_STRESS}/flood_${q}.result"))) +done +SEC9_DUR=$(($(date +%s) - SEC9_START)) +echo " Успешных: ${FLOOD_OK}/${QUEUE_FLOOD_COUNT} за ${SEC9_DUR}s" +check_ge "Queue-flood: >= 90% очередей работают" "$FLOOD_OK" "$((QUEUE_FLOOD_COUNT * 90 / 100))" + +# Проверяем ListQueues — должно быть >= 30 flood-очередей +R=$(sqs "$T1_AK" "$T1_SK" list-queues --queue-name-prefix "stress-flood" 2>&1) +LISTED=$(echo "$R" | grep -c "stress-flood" || true) +echo " ListQueues видит: $LISTED очередей" +check_ge "ListQueues >= ${QUEUE_FLOOD_COUNT}" "$LISTED" "$QUEUE_FLOOD_COUNT" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 10. MEMORY CHECK — RSS до нагрузки +# ═══════════════════════════════════════════════════════════ +echo "═══ 10. Memory check — запоминаем RSS ═══" +RSS_BEFORE=$(get_pod_rss_mb) +echo " RSS до тяжёлой фазы: ${RSS_BEFORE} MB" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 11. KILL POD + восстановление из Redis +# ═══════════════════════════════════════════════════════════ +echo "═══ 11. Kill pod → восстановление из Redis ═══" + +# Перед kill — отправляем в burst-очередь ещё 20 msg +EXTRA=20 +for m in $(seq 1 $EXTRA); do + sqs "$T1_AK" "$T1_SK" send-message --queue-url "$BURST_QURL" \ + --message-body "pre-kill-${m}-${TS}" >/dev/null 2>&1 + if (( m % 5 == 0 )); then printf "[pre-kill:${m}] " >&2; fi +done +echo "" + +R=$(sqs "$T1_AK" "$T1_SK" get-queue-attributes --queue-url "$BURST_QURL" \ + --attribute-names ApproximateNumberOfMessages 2>&1) +BEFORE_KILL=$(echo "$R" | grep -oP '"ApproximateNumberOfMessages"\s*:\s*"\K[0-9]+') +echo " Сообщений в burst до kill: ${BEFORE_KILL:-?}" + +kill_pod +echo " Ждём рестарт..." +wait_for_pod_ready +wait_for_healthy + +sleep 3 +R=$(sqs "$T1_AK" "$T1_SK" get-queue-attributes --queue-url "$BURST_QURL" \ + --attribute-names ApproximateNumberOfMessages 2>&1) +AFTER_KILL=$(echo "$R" | grep -oP '"ApproximateNumberOfMessages"\s*:\s*"\K[0-9]+') +echo " Сообщений после рестарта: ${AFTER_KILL:-0}" +check_eq "Данные восстановлены из Redis" "${AFTER_KILL:-0}" "${BEFORE_KILL}" + +R=$(admin_api GET "/admin/tenants") +TC=$(echo "$R" | grep -c "stress-t1-${TS}" || true) +check_ge "Тенанты восстановлены" "$TC" "1" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 12. REDIS DISCONNECT / RECONNECT +# ═══════════════════════════════════════════════════════════ +echo "═══ 12. Redis disconnect simulation ═══" + +cat <<'NETPOL' | kubectl apply -n "$NAMESPACE" -f - 2>&1 | head -1 +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: stress-test-block-redis +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: shared-sqs + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + ports: + - protocol: TCP + port: 443 + - protocol: TCP + port: 80 + - to: + - namespaceSelector: {} + ports: + - protocol: TCP + port: 53 + - protocol: UDP + port: 53 +NETPOL +echo " NetworkPolicy applied — Redis заблокирован" +sleep 5 + +R_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \ + -H "Authorization: Bearer $ADMIN_TOKEN" "${BASE_URL}/admin/tenants" 2>/dev/null) +echo " HTTP при потере Redis: $R_CODE" +TOTAL=$((TOTAL + 1)) +if [[ "$R_CODE" =~ ^[2345][0-9][0-9]$ ]]; then + PASS=$((PASS + 1)); echo " ✅ Сервис отвечает (HTTP $R_CODE)" +else + FAIL=$((FAIL + 1)); echo " ❌ Сервис не отвечает (HTTP $R_CODE)" +fi + +kubectl delete networkpolicy stress-test-block-redis -n "$NAMESPACE" 2>&1 | head -1 +echo " NetworkPolicy удалена — Redis доступен" +sleep 5 + +R=$(sqs "$T1_AK" "$T1_SK" send-message --queue-url "$BURST_QURL" \ + --message-body "after-redis-recovery-${TS}" 2>&1) +check "Сервис работает после Redis reconnect" "$R" "MessageId" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 13. СМЕШАННАЯ НАГРУЗКА — send + receive + delete + getattr (20s) +# ═══════════════════════════════════════════════════════════ +echo "═══ 13. Смешанная нагрузка: 10 senders + 10 receivers + 3 getattr × ${MIXED_DURATION}s ═══" +SEC13_START=$(date +%s) + +MIXED_Q="stress-mixed-${TS}" +R=$(sqs "$T1_AK" "$T1_SK" create-queue --queue-name "$MIXED_Q") +MIXED_QURL=$(qurl "$T1_ID" "$MIXED_Q") + +# Предзагрузка 100 msg +for m in $(seq 1 100); do + sqs "$T1_AK" "$T1_SK" send-message --queue-url "$MIXED_QURL" \ + --message-body "mixed-init-${m}" >/dev/null 2>&1 + if (( m % 20 == 0 )); then printf "[init:${m}] " >&2; fi +done +echo "" +echo " Начальная загрузка: 100 msg" + +# 10 Отправителей +for w in $(seq 1 10); do + ( + end=$((SECONDS + MIXED_DURATION)); ok=0 + while [[ $SECONDS -lt $end ]]; do + R=$(sqs "$T1_AK" "$T1_SK" send-message --queue-url "$MIXED_QURL" \ + --message-body "mixed-s${w}-$(date +%s%N)" 2>&1) + if echo "$R" | grep -q "MessageId"; then ok=$((ok + 1)); fi + done + echo "$ok" > "${TMPDIR_STRESS}/mixed_send_${w}.result" + ) & +done + +# 10 Читателей-удалителей +for w in $(seq 1 10); do + ( + end=$((SECONDS + MIXED_DURATION)); recv=0 + while [[ $SECONDS -lt $end ]]; do + R=$(sqs "$T1_AK" "$T1_SK" receive-message --queue-url "$MIXED_QURL" \ + --max-number-of-messages 10 --wait-time-seconds 1 2>&1) + HANDLES=$(echo "$R" | grep -oP '"ReceiptHandle"\s*:\s*"\K[^"]+') + while IFS= read -r handle; do + [[ -z "$handle" ]] && continue + sqs "$T1_AK" "$T1_SK" delete-message --queue-url "$MIXED_QURL" \ + --receipt-handle "$handle" >/dev/null 2>&1 + recv=$((recv + 1)) + done <<< "$HANDLES" + done + echo "$recv" > "${TMPDIR_STRESS}/mixed_recv_${w}.result" + ) & +done + +# 3 GetQueueAttributes +for w in $(seq 1 3); do + ( + end=$((SECONDS + MIXED_DURATION)); ok=0; fail=0 + while [[ $SECONDS -lt $end ]]; do + R=$(sqs "$T1_AK" "$T1_SK" get-queue-attributes --queue-url "$MIXED_QURL" \ + --attribute-names All 2>&1) + if echo "$R" | grep -q "ApproximateNumberOfMessages"; then ok=$((ok + 1)); else fail=$((fail + 1)); fi + sleep 0.5 + done + echo "${ok}:${fail}" > "${TMPDIR_STRESS}/mixed_attr_${w}.result" + ) & +done + +# Прогресс каждые 5 сек +( + elapsed=0 + while [[ $elapsed -lt $MIXED_DURATION ]]; do + sleep 5 + elapsed=$((elapsed + 5)) + printf "[mixed: ${elapsed}/${MIXED_DURATION}s] " >&2 + done +) & +PROGRESS_PID=$! + +echo " ⏳ Смешанная нагрузка ${MIXED_DURATION}s..." +wait +echo "" + +MIXED_SENT=0; MIXED_RECV=0; ATTR_OK=0; ATTR_FAIL=0 +for w in $(seq 1 10); do + [[ -f "${TMPDIR_STRESS}/mixed_send_${w}.result" ]] && MIXED_SENT=$((MIXED_SENT + $(cat "${TMPDIR_STRESS}/mixed_send_${w}.result"))) + [[ -f "${TMPDIR_STRESS}/mixed_recv_${w}.result" ]] && MIXED_RECV=$((MIXED_RECV + $(cat "${TMPDIR_STRESS}/mixed_recv_${w}.result"))) +done +for w in $(seq 1 3); do + if [[ -f "${TMPDIR_STRESS}/mixed_attr_${w}.result" ]]; then + IFS=':' read -r ok fail < "${TMPDIR_STRESS}/mixed_attr_${w}.result" + ATTR_OK=$((ATTR_OK + ok)); ATTR_FAIL=$((ATTR_FAIL + fail)) + fi +done +SEC13_DUR=$(($(date +%s) - SEC13_START)) +echo " Отправлено: $MIXED_SENT, прочитано+удалено: $MIXED_RECV | Attr: ok=$ATTR_OK fail=$ATTR_FAIL | ${SEC13_DUR}s" +check_ge "Mixed: отправлено > 0" "$MIXED_SENT" "1" +check_ge "Mixed: прочитано > 0" "$MIXED_RECV" "1" +check_eq "Mixed: GetQueueAttributes 0 сбоев" "$ATTR_FAIL" "0" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 14. MEMORY CHECK — после нагрузки +# ═══════════════════════════════════════════════════════════ +echo "═══ 14. Memory check — RSS после нагрузки ═══" +RSS_AFTER=$(get_pod_rss_mb) +echo " RSS до: ${RSS_BEFORE} MB, после: ${RSS_AFTER} MB" +if [[ "$RSS_BEFORE" -gt 0 && "$RSS_AFTER" -gt 0 ]]; then + RSS_GROWTH=$((RSS_AFTER - RSS_BEFORE)) + echo " Рост: ${RSS_GROWTH} MB" + # Допускаем рост до 100 MB — выше уже подозрительно + check_le "Рост RSS <= 100MB" "$RSS_GROWTH" "100" +else + echo " ⚠️ Не удалось измерить RSS (возможно /proc недоступен)" + TOTAL=$((TOTAL + 1)); PASS=$((PASS + 1)) + echo " ✅ Memory check пропущен (нет доступа к /proc)" +fi +echo "" + +# ═══════════════════════════════════════════════════════════ +# 15. MULTI-KILL: kill → up → kill → up +# ═══════════════════════════════════════════════════════════ +echo "═══ 15. Multi-kill: 3 рестарта подряд ═══" +SEC15_START=$(date +%s) + +# Отправляем маркерное сообщение +MARKER_Q="stress-marker-${TS}" +R=$(sqs "$T1_AK" "$T1_SK" create-queue --queue-name "$MARKER_Q") +MARKER_QURL=$(qurl "$T1_ID" "$MARKER_Q") +sqs "$T1_AK" "$T1_SK" send-message --queue-url "$MARKER_QURL" \ + --message-body "marker-survive-3-kills-${TS}" >/dev/null 2>&1 + +for k in 1 2 3; do + echo " --- Kill #${k} ---" + kill_pod + wait_for_pod_ready + wait_for_healthy + echo " Kill #${k}: под восстановлен ✓" +done + +# Маркер выжил? +R=$(sqs "$T1_AK" "$T1_SK" receive-message --queue-url "$MARKER_QURL" \ + --max-number-of-messages 1 --wait-time-seconds 2 2>&1) +SEC15_DUR=$(($(date +%s) - SEC15_START)) +echo " 3 рестарта за ${SEC15_DUR}s" +check "Маркер выжил 3 рестарта" "$R" "marker-survive-3-kills" +echo "" + +# ═══════════════════════════════════════════════════════════ +# 16. CLEANUP +# ═══════════════════════════════════════════════════════════ +echo "═══ 16. Cleanup ═══" +echo " Удаляю очереди..." + +# Основные очереди +for i in $(seq 1 $TENANT_COUNT); do + tak=$(cat "${TMPDIR_STRESS}/tak_${i}" 2>/dev/null) + tsk=$(cat "${TMPDIR_STRESS}/tsk_${i}" 2>/dev/null) + tid=$(cat "${TMPDIR_STRESS}/tid_${i}" 2>/dev/null) + [[ -n "$tak" ]] && sqs "$tak" "$tsk" delete-queue --queue-url "${BASE_URL}/${tid}/${STRESS_Q}" >/dev/null 2>&1 +done +sqs "$T1_AK" "$T1_SK" delete-queue --queue-url "$BURST_QURL" >/dev/null 2>&1 +sqs "$T1_AK" "$T1_SK" delete-queue --queue-url "$MIXED_QURL" >/dev/null 2>&1 +sqs "$T1_AK" "$T1_SK" delete-queue --queue-url "$LONGPOLL_QURL" >/dev/null 2>&1 +sqs "$T1_AK" "$T1_SK" delete-queue --queue-url "$RACE_QURL" >/dev/null 2>&1 +sqs "$T1_AK" "$T1_SK" delete-queue --queue-url "$BATCH_QURL" >/dev/null 2>&1 +sqs "$T1_AK" "$T1_SK" delete-queue --queue-url "$MARKER_QURL" >/dev/null 2>&1 + +# Flood-очереди +for q in $(seq 1 $QUEUE_FLOOD_COUNT); do + sqs "$T1_AK" "$T1_SK" delete-queue \ + --queue-url "$(qurl "$T1_ID" "stress-flood-${q}-${TS}")" >/dev/null 2>&1 & +done +wait + +echo " Очереди удалены" + +TOTAL_DUR=$(($(date +%s) - STARTED_AT)) +echo "" +echo "╔═══════════════════════════════════════════════════════════════════╗" +echo "║ ИТОГО: ${PASS}/${TOTAL} ✅ ${FAIL}/${TOTAL} ❌ ║" +echo "║ Время: ${TOTAL_DUR}s ║" +echo "╚═══════════════════════════════════════════════════════════════════╝" + +if [[ $FAIL -gt 0 ]]; then exit 1; fi