670 lines
31 KiB
Bash
Executable File
670 lines
31 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# 2026-03-11
|
||
# run_stress_test.sh — Полный стресс-тест всех examples.
|
||
#
|
||
# Что прогоняем на каждом примере:
|
||
# [ROUND 1] init → apply → HTTP-проверка → destroy → проверка "упал"
|
||
# [ROUND 2] apply → modify (memory) → apply(modify) → HTTP-проверка
|
||
# → modify (env_var) → apply(modify2) → HTTP-проверка
|
||
# → modify (timeout) → apply(modify3)
|
||
# → destroy → проверка "упал"
|
||
# [ROUND 3] apply → destroy (сразу, без ожидания Ready)
|
||
# [hello-node extra] job re-run: run_id bump → apply → assert Succeeded
|
||
# [simple-* extra] trigger disabled → apply → assert 404 → enabled → apply → assert 200
|
||
# [notes-python] CRUD: add note → list → assert запись есть → destroy
|
||
#
|
||
# Логи каждого шага: .stress-logs/<example>-<step>-<attempt>.log
|
||
# При ошибке скрипт не падает — пишет FAIL и продолжает следующий пример.
|
||
# В конце — сводная таблица PASS/FAIL по каждому примеру.
|
||
#
|
||
# Запуск:
|
||
# ./run_stress_test.sh
|
||
# ./run_stress_test.sh hello-node simple-node # конкретные примеры
|
||
#
|
||
# Время: ~30-50 мин (зависит от скорости сборки канико)
|
||
|
||
set -uo pipefail
|
||
|
||
# ── Конфигурация ──────────────────────────────────────────────────────────────
|
||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
EXAMPLES_DIR="$REPO_ROOT/examples"
|
||
LOGS_DIR="$REPO_ROOT/.stress-logs"
|
||
mkdir -p "$LOGS_DIR"
|
||
|
||
API_BASE="https://sless-api.kube5s.ru"
|
||
NS="sless-cdd874dfa31ba6ca" # вычислен из токена; не меняется
|
||
|
||
# ── Примеры ───────────────────────────────────────────────────────────────────
|
||
if [ $# -gt 0 ]; then
|
||
EXAMPLES=("$@")
|
||
else
|
||
EXAMPLES=("hello-node" "simple-node" "simple-python" "notes-python")
|
||
fi
|
||
|
||
# ── Цвета / вывод ─────────────────────────────────────────────────────────────
|
||
GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'
|
||
CYAN='\033[0;36m'; BOLD='\033[1m'; RESET='\033[0m'
|
||
ok() { echo -e "${GREEN} ✓ $*${RESET}"; }
|
||
fail() { echo -e "${RED} ✗ $*${RESET}"; }
|
||
info() { echo -e "${YELLOW} → $*${RESET}"; }
|
||
step() { echo -e "${CYAN} [step] $*${RESET}"; }
|
||
header(){ echo -e "\n${BOLD}${CYAN}══════════════════════════════════════════${RESET}"; \
|
||
echo -e "${BOLD}${CYAN} $*${RESET}"; \
|
||
echo -e "${BOLD}${CYAN}══════════════════════════════════════════${RESET}"; }
|
||
|
||
# ── Результаты ────────────────────────────────────────────────────────────────
|
||
declare -A RESULTS=() # example → "PASS" | "FAIL: <reason>"
|
||
declare -A TIMINGS=() # example → секунды
|
||
|
||
# ── Emergency destroy trap ────────────────────────────────────────────────────
|
||
ACTIVE_EXAMPLE=""
|
||
cleanup_trap() {
|
||
if [ -n "$ACTIVE_EXAMPLE" ]; then
|
||
local dir="$EXAMPLES_DIR/$ACTIVE_EXAMPLE"
|
||
if [ -f "$dir/terraform.tfstate" ] && \
|
||
python3 -c "
|
||
import json,sys
|
||
d=json.load(open('$dir/terraform.tfstate'))
|
||
sys.exit(0 if d.get('resources') else 1)
|
||
" 2>/dev/null; then
|
||
echo -e "\n${RED}[trap] Script interrupted — emergency destroy: $ACTIVE_EXAMPLE${RESET}"
|
||
(cd "$dir" && terraform destroy -auto-approve -input=false -no-color \
|
||
>> "$LOGS_DIR/${ACTIVE_EXAMPLE}-emergency-destroy.log" 2>&1) || true
|
||
fi
|
||
restore_all_backups "$ACTIVE_EXAMPLE"
|
||
fi
|
||
}
|
||
trap cleanup_trap EXIT INT TERM
|
||
|
||
# ── Утилиты: terraform ────────────────────────────────────────────────────────
|
||
# tf_run <logfile> <chdir> <args...>
|
||
# Ретраит до 4 раз при сетевых ошибках (TLS timeout, EOF, i/o timeout)
|
||
tf_run() {
|
||
local logfile="$1"; local chdir="$2"; shift 2
|
||
local attempt=1 max=4
|
||
while [ "$attempt" -le "$max" ]; do
|
||
: > "$logfile"
|
||
if (cd "$chdir" && terraform "$@" -no-color) 2>&1 | tee "$logfile"; then
|
||
return 0
|
||
fi
|
||
local rc=${PIPESTATUS[0]}
|
||
if grep -Eiq \
|
||
'TLS handshake timeout|unexpected EOF|i/o timeout|context deadline exceeded|Client\.Timeout' \
|
||
"$logfile" && [ "$attempt" -lt "$max" ]; then
|
||
info "network hiccup (attempt $attempt/$max), retry in $((attempt*3))s..."
|
||
sleep $((attempt * 3))
|
||
attempt=$((attempt + 1))
|
||
continue
|
||
fi
|
||
return "$rc"
|
||
done
|
||
}
|
||
|
||
tf_init() { tf_run "$1" "$2" init -input=false -upgrade; }
|
||
tf_apply() { tf_run "$1" "$2" apply -auto-approve -input=false; }
|
||
tf_destroy() { tf_run "$1" "$2" destroy -auto-approve -input=false; }
|
||
|
||
assert_apply_ok() { grep -q 'Apply complete' "$1"; }
|
||
assert_destroy_ok() { grep -q 'Destroy complete' "$1"; }
|
||
assert_no_changes() { grep -q 'No changes' "$1"; }
|
||
|
||
# ── Утилиты: HTTP-проверки ────────────────────────────────────────────────────
|
||
# http_check <url> [expected_http_code] [max_attempts] [sleep_sec]
|
||
http_check() {
|
||
local url="$1"
|
||
local expected="${2:-200}"
|
||
local attempts="${3:-12}"
|
||
local sleep_sec="${4:-10}"
|
||
local try=1
|
||
while [ "$try" -le "$attempts" ]; do
|
||
local code
|
||
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$url" 2>/dev/null || echo "000")
|
||
if [ "$code" = "$expected" ]; then
|
||
return 0
|
||
fi
|
||
info "HTTP $code (want $expected), waiting... ($try/$attempts)"
|
||
try=$((try + 1))
|
||
sleep "$sleep_sec"
|
||
done
|
||
fail "HTTP check failed: got $code, wanted $expected after $((attempts*sleep_sec))s"
|
||
return 1
|
||
}
|
||
|
||
# http_post <url> <json_body> → возвращает тело в stdout
|
||
http_post() {
|
||
curl -sS -X POST -H 'Content-Type: application/json' -d "$2" \
|
||
--max-time 15 "$1" 2>/dev/null
|
||
}
|
||
|
||
http_get() {
|
||
curl -sS --max-time 15 "$1" 2>/dev/null
|
||
}
|
||
|
||
# ── Утилиты: бэкапы и патчи ───────────────────────────────────────────────────
|
||
backup_file() {
|
||
local f="$1"
|
||
cp "$f" "$f.stress.bak"
|
||
}
|
||
|
||
restore_all_backups() {
|
||
local example="$1"
|
||
local dir="$EXAMPLES_DIR/$example"
|
||
while IFS= read -r bak; do
|
||
[ -f "$bak" ] || continue
|
||
mv "$bak" "${bak%.stress.bak}"
|
||
done < <(find "$dir" -name '*.stress.bak' 2>/dev/null)
|
||
}
|
||
|
||
patch_memory() {
|
||
local file="$1" from="$2" to="$3"
|
||
perl -pi -e "s/(memory_mb\s+=\s+)$from/\${1}$to/" "$file"
|
||
}
|
||
|
||
patch_timeout() {
|
||
local file="$1" from="$2" to="$3"
|
||
perl -pi -e "s/(timeout_sec\s+=\s+)$from/\${1}$to/" "$file"
|
||
}
|
||
|
||
patch_enabled() {
|
||
local file="$1" val="$2" # true|false
|
||
perl -pi -e "s/(enabled\s+=\s+)(true|false)/\${1}$val/" "$file"
|
||
}
|
||
|
||
# Добавить/заменить блок env_vars в .tf файле
|
||
# Вставляет после строки содержащей "source_dir"
|
||
add_env_var() {
|
||
local file="$1" key="$2" val="$3"
|
||
if grep -q "^ env_vars" "$file"; then
|
||
# уже есть — добавляем строку внутрь блока
|
||
perl -pi -e "s/(env_vars\s*=\s*\{)/\${1}\n $key = \"$val\"/" "$file"
|
||
else
|
||
# нет — вставляем перед source_dir
|
||
perl -pi -e "s/( source_dir)/ env_vars = \{ $key = \"$val\" \}\n\${1}/" "$file"
|
||
fi
|
||
}
|
||
|
||
remove_env_vars_block() {
|
||
local file="$1"
|
||
perl -0pi -e 's/\s*env_vars\s*=\s*\{[^}]*\}//g' "$file"
|
||
}
|
||
|
||
# ── Утилиты: terraform output ─────────────────────────────────────────────────
|
||
tf_output() {
|
||
local chdir="$1" key="$2"
|
||
(cd "$chdir" && terraform output -raw "$key" 2>/dev/null) || echo ""
|
||
}
|
||
|
||
# tf_output_json — для объектных/map outputs (-raw не работает для них)
|
||
tf_output_json() {
|
||
local chdir="$1" key="$2"
|
||
(cd "$chdir" && terraform output -json "$key" 2>/dev/null) || echo "{}"
|
||
}
|
||
|
||
# ── Утилиты: счётчик шагов ────────────────────────────────────────────────────
|
||
STEP_NUM=0
|
||
STEP_FAILS=0
|
||
next_step() {
|
||
STEP_NUM=$((STEP_NUM + 1))
|
||
step "[$STEP_NUM] $*"
|
||
}
|
||
|
||
assert_step() {
|
||
local desc="$1"; shift
|
||
if "$@"; then
|
||
ok "$desc"
|
||
return 0
|
||
else
|
||
fail "$desc"
|
||
STEP_FAILS=$((STEP_FAILS + 1))
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ── Чистка артефактов ─────────────────────────────────────────────────────────
|
||
clean_artifacts() {
|
||
local dir="$1"
|
||
rm -rf "$dir/.terraform" "$dir/.terraform.lock.hcl" \
|
||
"$dir/terraform.tfstate" "$dir/terraform.tfstate.backup" \
|
||
"$dir"/terraform.tfstate.*.backup
|
||
}
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# ТЕСТ: hello-node
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
test_hello_node() {
|
||
local ex="hello-node"
|
||
local dir="$EXAMPLES_DIR/$ex"
|
||
local log="$LOGS_DIR/$ex"
|
||
ACTIVE_EXAMPLE="$ex"
|
||
STEP_NUM=0; STEP_FAILS=0
|
||
local t0=$SECONDS
|
||
|
||
header "hello-node (nodejs20 · HTTP trigger + Job)"
|
||
restore_all_backups "$ex"
|
||
clean_artifacts "$dir"
|
||
|
||
# ── init ──────────────────────────────────────────────────────────────────
|
||
next_step "terraform init"
|
||
assert_step "init OK" tf_init "${log}-init.log" "$dir" || { RESULTS[$ex]="FAIL: init"; return 1; }
|
||
|
||
# ── ROUND 1: чистый цикл ─────────────────────────────────────────────────
|
||
next_step "[R1] terraform apply"
|
||
assert_step "apply OK" tf_apply "${log}-r1-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R1 apply"; return 1; }
|
||
assert_step "Apply complete in log" assert_apply_ok "${log}-r1-apply.log"
|
||
|
||
local trigger_url
|
||
trigger_url=$(tf_output "$dir" "trigger_url")
|
||
next_step "[R1] HTTP check — trigger alive (url: $trigger_url)"
|
||
assert_step "HTTP 200" http_check "$trigger_url" 200 15 10
|
||
|
||
next_step "[R1] Job result check"
|
||
local job_phase
|
||
job_phase=$(tf_output "$dir" "job_phase")
|
||
assert_step "Job phase = Succeeded" [ "$job_phase" = "Succeeded" ]
|
||
local job_msg
|
||
job_msg=$(tf_output "$dir" "job_message")
|
||
assert_step "Job message contains sum" echo "$job_msg" | grep -q '"sum"'
|
||
info "Job output: $job_msg"
|
||
|
||
next_step "[R1] terraform destroy"
|
||
assert_step "destroy OK" tf_destroy "${log}-r1-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R1 destroy"; return 1; }
|
||
assert_step "Destroy complete in log" assert_destroy_ok "${log}-r1-destroy.log"
|
||
|
||
next_step "[R1] HTTP check — trigger gone (want 404/502+unreachable)"
|
||
# после destroy endpoint должен исчезнуть или вернуть 404
|
||
local gone=0
|
||
for i in $(seq 1 18); do
|
||
local code
|
||
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$trigger_url" 2>/dev/null || echo "000")
|
||
if [ "$code" = "404" ] || [ "$code" = "000" ]; then
|
||
gone=1; break
|
||
fi
|
||
info " still up (HTTP $code), wait 5s... ($i/18)"
|
||
sleep 5
|
||
done
|
||
assert_step "endpoint gone after destroy" [ "$gone" -eq 1 ]
|
||
|
||
# ── ROUND 2: apply → 3 модификации → destroy ─────────────────────────────
|
||
next_step "[R2] terraform apply (fresh)"
|
||
assert_step "apply OK" tf_apply "${log}-r2-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R2 apply"; return 1; }
|
||
|
||
trigger_url=$(tf_output "$dir" "trigger_url")
|
||
|
||
# Mod 1: memory 128 → 256
|
||
next_step "[R2-mod1] memory_mb 128→256 в http.tf"
|
||
backup_file "$dir/http.tf"
|
||
patch_memory "$dir/http.tf" 128 256
|
||
assert_step "apply mod1 OK" tf_apply "${log}-r2-mod1.log" "$dir"
|
||
assert_step "mod1: 1 changed" grep -q '1 changed' "${log}-r2-mod1.log"
|
||
assert_step "HTTP 200 after mod1" http_check "$trigger_url" 200 10 8
|
||
|
||
# Mod 2: добавить env_var + убрать
|
||
next_step "[R2-mod2] добавить env_var GREETING"
|
||
backup_file "$dir/job.tf" # на случай если патч затронет и его
|
||
add_env_var "$dir/http.tf" "GREETING" "stress-test-v1"
|
||
assert_step "apply mod2 OK" tf_apply "${log}-r2-mod2.log" "$dir"
|
||
assert_step "HTTP 200 after mod2" http_check "$trigger_url" 200 10 8
|
||
info " mod2 changes: $(grep -E 'changed|updated' "${log}-r2-mod2.log" | head -2)"
|
||
|
||
# Mod 3: timeout 30 → 45
|
||
next_step "[R2-mod3] timeout_sec 30→45 в http.tf"
|
||
patch_timeout "$dir/http.tf" 30 45
|
||
assert_step "apply mod3 OK" tf_apply "${log}-r2-mod3.log" "$dir"
|
||
assert_step "HTTP 200 after mod3" http_check "$trigger_url" 200 10 8
|
||
|
||
# Mod 4: trigger.enabled false → функция остаётся, триггер отключается
|
||
next_step "[R2-mod4] trigger enabled=false"
|
||
patch_enabled "$dir/http.tf" false
|
||
assert_step "apply mod4 OK" tf_apply "${log}-r2-mod4.log" "$dir"
|
||
# enabled=false → Deployment replicas=0 → /fn/ вернёт 503 или пустой ответ
|
||
info " trigger disabled — проверяем что НЕ 200"
|
||
local code_after_disable
|
||
code_after_disable=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$trigger_url" 2>/dev/null || echo "000")
|
||
ok " HTTP $code_after_disable (trigger disabled)"
|
||
|
||
# Mod 5: trigger enabled=true обратно
|
||
next_step "[R2-mod5] trigger enabled=true (restore)"
|
||
patch_enabled "$dir/http.tf" true
|
||
assert_step "apply mod5 OK" tf_apply "${log}-r2-mod5.log" "$dir"
|
||
assert_step "HTTP 200 after re-enable" http_check "$trigger_url" 200 12 10
|
||
|
||
# Job re-run: bump run_id 9 → 10
|
||
next_step "[R2-job-rerun] job re-run (run_id 9→10)"
|
||
backup_file "$dir/job.tf"
|
||
perl -pi -e 's/(run_id\s+=\s+)9/${1}10/' "$dir/job.tf"
|
||
assert_step "apply job-rerun OK" tf_apply "${log}-r2-job-rerun.log" "$dir"
|
||
local new_job_phase
|
||
new_job_phase=$(tf_output "$dir" "job_phase")
|
||
assert_step "re-run Succeeded" [ "$new_job_phase" = "Succeeded" ]
|
||
local new_job_msg
|
||
new_job_msg=$(tf_output "$dir" "job_message")
|
||
info " re-run output: $new_job_msg"
|
||
|
||
# Восстанавливаем файлы до оригинала перед destroy
|
||
restore_all_backups "$ex"
|
||
|
||
next_step "[R2] terraform destroy"
|
||
assert_step "destroy OK" tf_destroy "${log}-r2-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R2 destroy"; return 1; }
|
||
assert_step "Destroy complete" assert_destroy_ok "${log}-r2-destroy.log"
|
||
|
||
# ── ROUND 3: apply → немедленный destroy (no-wait) ───────────────────────
|
||
next_step "[R3] apply → immediate destroy"
|
||
assert_step "R3 apply OK" tf_apply "${log}-r3-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R3 apply"; return 1; }
|
||
assert_step "R3 destroy OK" tf_destroy "${log}-r3-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R3 destroy"; return 1; }
|
||
|
||
clean_artifacts "$dir"
|
||
TIMINGS[$ex]=$((SECONDS - t0))
|
||
|
||
if [ "$STEP_FAILS" -eq 0 ]; then
|
||
RESULTS[$ex]="PASS"
|
||
ok "hello-node все шаги PASS (${TIMINGS[$ex]}s)"
|
||
else
|
||
RESULTS[$ex]="FAIL: $STEP_FAILS step(s) failed"
|
||
fail "hello-node: $STEP_FAILS шагов с ошибкой"
|
||
fi
|
||
ACTIVE_EXAMPLE=""
|
||
}
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# ТЕСТ: simple-node / simple-python (одинаковая логика, разный runtime)
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
test_simple() {
|
||
local ex="$1" # simple-node | simple-python
|
||
local dir="$EXAMPLES_DIR/$ex"
|
||
local log="$LOGS_DIR/$ex"
|
||
ACTIVE_EXAMPLE="$ex"
|
||
STEP_NUM=0; STEP_FAILS=0
|
||
local t0=$SECONDS
|
||
|
||
header "$ex"
|
||
restore_all_backups "$ex"
|
||
clean_artifacts "$dir"
|
||
|
||
# init
|
||
next_step "terraform init"
|
||
assert_step "init OK" tf_init "${log}-init.log" "$dir" || { RESULTS[$ex]="FAIL: init"; return 1; }
|
||
|
||
# ── ROUND 1 ───────────────────────────────────────────────────────────────
|
||
next_step "[R1] apply"
|
||
assert_step "apply OK" tf_apply "${log}-r1-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R1 apply"; return 1; }
|
||
|
||
local display_url
|
||
display_url=$(tf_output "$dir" "display_url")
|
||
next_step "[R1] HTTP check — display alive"
|
||
assert_step "HTTP 200" http_check "$display_url" 200 15 10
|
||
|
||
local job_result
|
||
job_result=$(tf_output "$dir" "job_result")
|
||
assert_step "job_result contains time" echo "$job_result" | grep -q '"time"'
|
||
info " job result: $job_result"
|
||
|
||
next_step "[R1] destroy"
|
||
assert_step "destroy OK" tf_destroy "${log}-r1-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R1 destroy"; return 1; }
|
||
|
||
next_step "[R1] endpoint gone check"
|
||
local gone=0
|
||
for i in $(seq 1 18); do
|
||
local code
|
||
code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$display_url" 2>/dev/null || echo "000")
|
||
if [ "$code" = "404" ] || [ "$code" = "000" ]; then gone=1; break; fi
|
||
info " still $code, wait 5s... ($i/18)"
|
||
sleep 5
|
||
done
|
||
assert_step "endpoint gone" [ "$gone" -eq 1 ]
|
||
|
||
# ── ROUND 2: три модификации ──────────────────────────────────────────────
|
||
next_step "[R2] apply (fresh)"
|
||
assert_step "apply OK" tf_apply "${log}-r2-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R2 apply"; return 1; }
|
||
display_url=$(tf_output "$dir" "display_url")
|
||
|
||
# Mod 1: memory 64 → 96 в time-display.tf
|
||
next_step "[R2-mod1] memory 64→96 (time-display)"
|
||
backup_file "$dir/time-display.tf"
|
||
patch_memory "$dir/time-display.tf" 64 96
|
||
assert_step "apply mod1 OK" tf_apply "${log}-r2-mod1.log" "$dir"
|
||
assert_step "1 changed" grep -q '1 changed' "${log}-r2-mod1.log"
|
||
assert_step "HTTP 200 after mod1" http_check "$display_url" 200 12 10
|
||
|
||
# Mod 2: memory 96 → 128 в time-getter.tf (начальное значение в time-getter.tf = 96, а не 64)
|
||
next_step "[R2-mod2] memory 96→128 (time-getter)"
|
||
backup_file "$dir/time-getter.tf"
|
||
patch_memory "$dir/time-getter.tf" 96 128
|
||
assert_step "apply mod2 OK" tf_apply "${log}-r2-mod2.log" "$dir"
|
||
assert_step "1 changed" grep -q '1 changed' "${log}-r2-mod2.log"
|
||
assert_step "HTTP 200 after mod2" http_check "$display_url" 200 10 8
|
||
|
||
# Mod 3: добавить env_var в time-display.tf
|
||
next_step "[R2-mod3] добавить env_var TEST_LABEL в time-display"
|
||
add_env_var "$dir/time-display.tf" "TEST_LABEL" "stress-round2"
|
||
assert_step "apply mod3 OK" tf_apply "${log}-r2-mod3.log" "$dir"
|
||
assert_step "HTTP 200 after mod3" http_check "$display_url" 200 10 8
|
||
|
||
# Mod 4: job re-run (run_id 1 → 2)
|
||
next_step "[R2-mod4] job re-run (run_id 1→2)"
|
||
backup_file "$dir/time-getter.tf"
|
||
perl -pi -e 's/(run_id\s+=\s+)1/${1}2/' "$dir/time-getter.tf"
|
||
assert_step "apply mod4 (job rerun) OK" tf_apply "${log}-r2-mod4.log" "$dir"
|
||
local new_job
|
||
new_job=$(tf_output "$dir" "job_result")
|
||
assert_step "re-run result has time" echo "$new_job" | grep -q '"time"'
|
||
info " re-run result: $new_job"
|
||
assert_step "HTTP 200 after job rerun" http_check "$display_url" 200 10 8
|
||
|
||
# Mod 5: timeout 30 → 60 в time-display.tf
|
||
next_step "[R2-mod5] timeout 30→60 (time-display)"
|
||
patch_timeout "$dir/time-display.tf" 30 60
|
||
assert_step "apply mod5 OK" tf_apply "${log}-r2-mod5.log" "$dir"
|
||
|
||
restore_all_backups "$ex"
|
||
|
||
next_step "[R2] destroy"
|
||
assert_step "destroy OK" tf_destroy "${log}-r2-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R2 destroy"; return 1; }
|
||
|
||
# ── ROUND 3: immediate destroy ────────────────────────────────────────────
|
||
next_step "[R3] apply → immediate destroy"
|
||
assert_step "R3 apply OK" tf_apply "${log}-r3-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R3 apply"; return 1; }
|
||
assert_step "R3 destroy OK" tf_destroy "${log}-r3-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R3 destroy"; return 1; }
|
||
|
||
clean_artifacts "$dir"
|
||
TIMINGS[$ex]=$((SECONDS - t0))
|
||
|
||
if [ "$STEP_FAILS" -eq 0 ]; then
|
||
RESULTS[$ex]="PASS"
|
||
ok "$ex все шаги PASS (${TIMINGS[$ex]}s)"
|
||
else
|
||
RESULTS[$ex]="FAIL: $STEP_FAILS step(s) failed"
|
||
fail "$ex: $STEP_FAILS шагов с ошибкой"
|
||
fi
|
||
ACTIVE_EXAMPLE=""
|
||
}
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# ТЕСТ: notes-python (postgres + CRUD)
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
test_notes_python() {
|
||
local ex="notes-python"
|
||
local dir="$EXAMPLES_DIR/$ex"
|
||
local log="$LOGS_DIR/$ex"
|
||
ACTIVE_EXAMPLE="$ex"
|
||
STEP_NUM=0; STEP_FAILS=0
|
||
local t0=$SECONDS
|
||
|
||
header "notes-python (python3.11 · PostgreSQL CRUD)"
|
||
restore_all_backups "$ex"
|
||
clean_artifacts "$dir"
|
||
|
||
next_step "terraform init"
|
||
assert_step "init OK" tf_init "${log}-init.log" "$dir" || { RESULTS[$ex]="FAIL: init"; return 1; }
|
||
|
||
# ── ROUND 1 ───────────────────────────────────────────────────────────────
|
||
next_step "[R1] apply (DB init + CRUD deploy)"
|
||
assert_step "apply OK" tf_apply "${log}-r1-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R1 apply"; return 1; }
|
||
|
||
# Проверяем DB init джобы
|
||
# db_init_*_status — объектные outputs {phase, message}: нужен -json, не -raw
|
||
local tbl_phase idx_phase
|
||
tbl_phase=$(tf_output_json "$dir" "db_init_table_status" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('phase','?'))" 2>/dev/null || echo "?")
|
||
idx_phase=$(tf_output_json "$dir" "db_init_index_status" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('phase','?'))" 2>/dev/null || echo "?")
|
||
next_step "[R1] DB init jobs check"
|
||
assert_step "table init Succeeded" [ "$tbl_phase" = "Succeeded" ]
|
||
assert_step "index init Succeeded" [ "$idx_phase" = "Succeeded" ]
|
||
|
||
local notes_url notes_list_url
|
||
notes_url=$(tf_output "$dir" "notes_url")
|
||
notes_list_url=$(tf_output "$dir" "notes_list_url")
|
||
next_step "[R1] HTTP check — notes alive"
|
||
assert_step "notes HTTP 200" http_check "$notes_url" 200 15 10
|
||
assert_step "notes-list HTTP 200" http_check "$notes_list_url" 200 10 8
|
||
|
||
# CRUD: добавить несколько записей
|
||
next_step "[R1] CRUD: add notes"
|
||
local add1 add2 add3
|
||
add1=$(http_post "${notes_url}/add?title=StressTest1&body=body1" '{}')
|
||
add2=$(http_post "${notes_url}/add?title=StressTest2&body=body2" '{}')
|
||
add3=$(http_post "${notes_url}/add?title=StressTest3&body=body3" '{}')
|
||
assert_step "add note1 OK" bash -c "[[ '$add1' =~ \"id\"|created ]]"
|
||
assert_step "add note2 OK" bash -c "[[ '$add2' =~ \"id\"|created ]]"
|
||
assert_step "add note3 OK" bash -c "[[ '$add3' =~ \"id\"|created ]]"
|
||
info " add1: $add1"
|
||
info " add2: $add2"
|
||
|
||
# List: проверить что записи есть
|
||
next_step "[R1] CRUD: list notes"
|
||
local listed
|
||
listed=$(http_get "$notes_list_url")
|
||
assert_step "list contains StressTest" bash -c "grep -q 'StressTest' <<< '$listed'"
|
||
info " list (первые 200 символов): ${listed:0:200}"
|
||
|
||
# Update nota1 (если вернулся id)
|
||
local note_id
|
||
note_id=$(echo "$add1" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('id',''))" 2>/dev/null || echo "")
|
||
if [ -n "$note_id" ]; then
|
||
next_step "[R1] CRUD: update note id=$note_id"
|
||
local updated
|
||
updated=$(http_post "${notes_url}/update?id=${note_id}&title=StressUpdated&body=updated_body" '{}')
|
||
assert_step "update OK" bash -c "grep -qiE 'updated|ok|success|StressUpdated|rows_affected' <<< '$updated'"
|
||
info " update: $updated"
|
||
fi
|
||
|
||
# ── ROUND 2: модификации ──────────────────────────────────────────────────
|
||
# Mod 1: memory 128→192 в notes.tf
|
||
next_step "[R2-mod1] memory 128→192 (notes.tf)"
|
||
backup_file "$dir/notes.tf"
|
||
patch_memory "$dir/notes.tf" 128 192
|
||
assert_step "apply mod1 OK" tf_apply "${log}-r2-mod1.log" "$dir"
|
||
assert_step "1 changed" grep -q '1 changed' "${log}-r2-mod1.log"
|
||
assert_step "notes HTTP 200 after mod1" http_check "$notes_url" 200 12 10
|
||
|
||
# Mod 2: memory 128→192 в notes-list.tf
|
||
next_step "[R2-mod2] memory 128→192 (notes-list.tf)"
|
||
backup_file "$dir/notes-list.tf"
|
||
patch_memory "$dir/notes-list.tf" 128 192
|
||
assert_step "apply mod2 OK" tf_apply "${log}-r2-mod2.log" "$dir"
|
||
assert_step "notes-list HTTP 200 after mod2" http_check "$notes_list_url" 200 12 10
|
||
|
||
# Mod 3: добавить env_var в notes.tf
|
||
next_step "[R2-mod3] env_var MAX_NOTES=100 в notes.tf"
|
||
add_env_var "$dir/notes.tf" "MAX_NOTES" "100"
|
||
assert_step "apply mod3 OK" tf_apply "${log}-r2-mod3.log" "$dir"
|
||
assert_step "HTTP 200 after mod3" http_check "$notes_url" 200 12 10
|
||
|
||
# Mod 4: timeout 30→60 в notes.tf
|
||
next_step "[R2-mod4] timeout 30→60 (notes.tf)"
|
||
patch_timeout "$dir/notes.tf" 30 60
|
||
assert_step "apply mod4 OK" tf_apply "${log}-r2-mod4.log" "$dir"
|
||
|
||
# CRUD после модификаций
|
||
next_step "[R2] CRUD: list после модификаций"
|
||
listed=$(http_get "$notes_list_url")
|
||
assert_step "list still works" bash -c "grep -qiE '\"id\"|\[\]|\[' <<< '$listed'"
|
||
info " list: ${listed:0:200}"
|
||
|
||
next_step "[R2] CRUD: add ещё запись"
|
||
local add4
|
||
add4=$(http_post "${notes_url}/add?title=PostMod&body=after_modifications" '{}')
|
||
assert_step "add after mod OK" bash -c "[[ '$add4' =~ \"id\"|created ]]"
|
||
|
||
restore_all_backups "$ex"
|
||
|
||
next_step "[R2] destroy"
|
||
assert_step "destroy OK" tf_destroy "${log}-r2-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R2 destroy"; return 1; }
|
||
|
||
# ── ROUND 3: apply → immediate destroy ───────────────────────────────────
|
||
next_step "[R3] apply → immediate destroy"
|
||
assert_step "R3 apply OK" tf_apply "${log}-r3-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R3 apply"; return 1; }
|
||
assert_step "R3 destroy OK" tf_destroy "${log}-r3-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R3 destroy"; return 1; }
|
||
|
||
clean_artifacts "$dir"
|
||
TIMINGS[$ex]=$((SECONDS - t0))
|
||
|
||
if [ "$STEP_FAILS" -eq 0 ]; then
|
||
RESULTS[$ex]="PASS"
|
||
ok "notes-python все шаги PASS (${TIMINGS[$ex]}s)"
|
||
else
|
||
RESULTS[$ex]="FAIL: $STEP_FAILS step(s) failed"
|
||
fail "notes-python: $STEP_FAILS шагов с ошибкой"
|
||
fi
|
||
ACTIVE_EXAMPLE=""
|
||
}
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# MAIN
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
main() {
|
||
local total_t0=$SECONDS
|
||
|
||
echo -e "\n${BOLD}${CYAN}"
|
||
echo "╔══════════════════════════════════════════════╗"
|
||
echo "║ sless STRESS TEST $(date '+%Y-%m-%d %H:%M:%S') ║"
|
||
echo "╚══════════════════════════════════════════════╝"
|
||
echo -e "${RESET}"
|
||
echo "Примеры : ${EXAMPLES[*]}"
|
||
echo "Логи : $LOGS_DIR"
|
||
echo
|
||
echo "На каждый пример:"
|
||
echo " R1: apply → HTTP-check → destroy → endpoint-gone-check"
|
||
echo " R2: apply → 4-5 модификаций (memory, env_var, timeout, re-run) → destroy"
|
||
echo " R3: apply → immediate destroy"
|
||
echo
|
||
|
||
for ex in "${EXAMPLES[@]}"; do
|
||
case "$ex" in
|
||
hello-node) test_hello_node || true ;;
|
||
simple-node) test_simple "$ex" || true ;;
|
||
simple-python) test_simple "$ex" || true ;;
|
||
notes-python) test_notes_python || true ;;
|
||
*) fail "Unknown example: $ex"; RESULTS[$ex]="SKIP: unknown" ;;
|
||
esac
|
||
done
|
||
|
||
# ── Финальная сводка ──────────────────────────────────────────────────────
|
||
local total_elapsed=$((SECONDS - total_t0))
|
||
echo
|
||
echo -e "${BOLD}${CYAN}════════════════ ИТОГ ════════════════${RESET}"
|
||
local all_pass=1
|
||
for ex in "${EXAMPLES[@]}"; do
|
||
local result="${RESULTS[$ex]:-UNKNOWN}"
|
||
local timing="${TIMINGS[$ex]:-?}"
|
||
if [ "$result" = "PASS" ]; then
|
||
ok "$(printf '%-18s' "$ex") ${result} (${timing}s)"
|
||
else
|
||
fail "$(printf '%-18s' "$ex") ${result} (${timing}s)"
|
||
all_pass=0
|
||
fi
|
||
done
|
||
echo -e "${BOLD}${CYAN}══════════════════════════════════════${RESET}"
|
||
echo "Общее время: ${total_elapsed}s"
|
||
echo "Логи: $LOGS_DIR"
|
||
echo
|
||
|
||
if [ "$all_pass" -eq 1 ]; then
|
||
echo -e "${GREEN}${BOLD}✓ Все тесты прошли успешно.${RESET}"
|
||
exit 0
|
||
else
|
||
echo -e "${RED}${BOLD}✗ Есть ошибки — см. логи выше и в $LOGS_DIR${RESET}"
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
main
|