test: run_stress_test.sh — полный стресс-тест всех примеров

2026-03-11 11:14:06 +04:00
parent 6c6040d8f8
commit 3026d032b3
15 changed files with 1492 additions and 0 deletions
@@ -0,0 +1,662 @@
+#!/usr/bin/env bash
+# 2026-03-11
+# run_stress_test.sh — Полный стресс-тест всех examples.
+#
+# Что прогоняем на каждом примере:
+#   [ROUND 1] init → apply → HTTP-проверка → destroy → проверка "упал"
+#   [ROUND 2] apply → modify (memory) → apply(modify) → HTTP-проверка
+#             → modify (env_var) → apply(modify2) → HTTP-проверка
+#             → modify (timeout) → apply(modify3)
+#             → destroy → проверка "упал"
+#   [ROUND 3] apply → destroy (сразу, без ожидания Ready)
+#   [hello-node extra] job re-run: run_id bump → apply → assert Succeeded
+#   [simple-* extra]   trigger disabled → apply → assert 404 → enabled → apply → assert 200
+#   [notes-python]     CRUD: add note → list → assert запись есть → destroy
+#
+# Логи каждого шага: .stress-logs/<example>-<step>-<attempt>.log
+# При ошибке скрипт не падает — пишет FAIL и продолжает следующий пример.
+# В конце — сводная таблица PASS/FAIL по каждому примеру.
+#
+# Запуск:
+#   ./run_stress_test.sh
+#   ./run_stress_test.sh hello-node simple-node   # конкретные примеры
+#
+# Время: ~30-50 мин (зависит от скорости сборки канико)
+
+set -uo pipefail
+
+# ── Конфигурация ──────────────────────────────────────────────────────────────
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+EXAMPLES_DIR="$REPO_ROOT/examples"
+LOGS_DIR="$REPO_ROOT/.stress-logs"
+mkdir -p "$LOGS_DIR"
+
+API_BASE="https://sless-api.kube5s.ru"
+NS="sless-cdd874dfa31ba6ca"   # вычислен из токена; не меняется
+
+# ── Примеры ───────────────────────────────────────────────────────────────────
+if [ $# -gt 0 ]; then
+  EXAMPLES=("$@")
+else
+  EXAMPLES=("hello-node" "simple-node" "simple-python" "notes-python")
+fi
+
+# ── Цвета / вывод ─────────────────────────────────────────────────────────────
+GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'
+CYAN='\033[0;36m'; BOLD='\033[1m'; RESET='\033[0m'
+ok()    { echo -e "${GREEN}    ✓ $*${RESET}"; }
+fail()  { echo -e "${RED}    ✗ $*${RESET}"; }
+info()  { echo -e "${YELLOW}    → $*${RESET}"; }
+step()  { echo -e "${CYAN}  [step] $*${RESET}"; }
+header(){ echo -e "\n${BOLD}${CYAN}══════════════════════════════════════════${RESET}"; \
+          echo -e "${BOLD}${CYAN}  $*${RESET}"; \
+          echo -e "${BOLD}${CYAN}══════════════════════════════════════════${RESET}"; }
+
+# ── Результаты ────────────────────────────────────────────────────────────────
+declare -A RESULTS=()     # example → "PASS" | "FAIL: <reason>"
+declare -A TIMINGS=()     # example → секунды
+
+# ── Emergency destroy trap ────────────────────────────────────────────────────
+ACTIVE_EXAMPLE=""
+cleanup_trap() {
+  if [ -n "$ACTIVE_EXAMPLE" ]; then
+    local dir="$EXAMPLES_DIR/$ACTIVE_EXAMPLE"
+    if [ -f "$dir/terraform.tfstate" ] && \
+       python3 -c "
+import json,sys
+d=json.load(open('$dir/terraform.tfstate'))
+sys.exit(0 if d.get('resources') else 1)
+" 2>/dev/null; then
+      echo -e "\n${RED}[trap] Script interrupted — emergency destroy: $ACTIVE_EXAMPLE${RESET}"
+      (cd "$dir" && terraform destroy -auto-approve -input=false -no-color \
+        >> "$LOGS_DIR/${ACTIVE_EXAMPLE}-emergency-destroy.log" 2>&1) || true
+    fi
+    restore_all_backups "$ACTIVE_EXAMPLE"
+  fi
+}
+trap cleanup_trap EXIT INT TERM
+
+# ── Утилиты: terraform ────────────────────────────────────────────────────────
+# tf_run <logfile> <chdir> <args...>
+# Ретраит до 4 раз при сетевых ошибках (TLS timeout, EOF, i/o timeout)
+tf_run() {
+  local logfile="$1"; local chdir="$2"; shift 2
+  local attempt=1 max=4
+  while [ "$attempt" -le "$max" ]; do
+    : > "$logfile"
+    if (cd "$chdir" && terraform "$@" -no-color) 2>&1 | tee "$logfile"; then
+      return 0
+    fi
+    local rc=${PIPESTATUS[0]}
+    if grep -Eiq \
+      'TLS handshake timeout|unexpected EOF|i/o timeout|context deadline exceeded|Client\.Timeout' \
+      "$logfile" && [ "$attempt" -lt "$max" ]; then
+      info "network hiccup (attempt $attempt/$max), retry in $((attempt*3))s..."
+      sleep $((attempt * 3))
+      attempt=$((attempt + 1))
+      continue
+    fi
+    return "$rc"
+  done
+}
+
+tf_init()    { tf_run "$1" "$2" init    -input=false -upgrade; }
+tf_apply()   { tf_run "$1" "$2" apply   -auto-approve -input=false; }
+tf_destroy() { tf_run "$1" "$2" destroy -auto-approve -input=false; }
+
+assert_apply_ok()   { grep -q 'Apply complete'   "$1"; }
+assert_destroy_ok() { grep -q 'Destroy complete' "$1"; }
+assert_no_changes() { grep -q 'No changes'       "$1"; }
+
+# ── Утилиты: HTTP-проверки ────────────────────────────────────────────────────
+# http_check <url> [expected_http_code] [max_attempts] [sleep_sec]
+http_check() {
+  local url="$1"
+  local expected="${2:-200}"
+  local attempts="${3:-12}"
+  local sleep_sec="${4:-10}"
+  local try=1
+  while [ "$try" -le "$attempts" ]; do
+    local code
+    code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$url" 2>/dev/null || echo "000")
+    if [ "$code" = "$expected" ]; then
+      return 0
+    fi
+    info "HTTP $code (want $expected), waiting... ($try/$attempts)"
+    try=$((try + 1))
+    sleep "$sleep_sec"
+  done
+  fail "HTTP check failed: got $code, wanted $expected after $((attempts*sleep_sec))s"
+  return 1
+}
+
+# http_post <url> <json_body>  → возвращает тело в stdout
+http_post() {
+  curl -sS -X POST -H 'Content-Type: application/json' -d "$2" \
+    --max-time 15 "$1" 2>/dev/null
+}
+
+http_get() {
+  curl -sS --max-time 15 "$1" 2>/dev/null
+}
+
+# ── Утилиты: бэкапы и патчи ───────────────────────────────────────────────────
+backup_file() {
+  local f="$1"
+  cp "$f" "$f.stress.bak"
+}
+
+restore_all_backups() {
+  local example="$1"
+  local dir="$EXAMPLES_DIR/$example"
+  while IFS= read -r bak; do
+    [ -f "$bak" ] || continue
+    mv "$bak" "${bak%.stress.bak}"
+  done < <(find "$dir" -name '*.stress.bak' 2>/dev/null)
+}
+
+patch_memory() {
+  local file="$1" from="$2" to="$3"
+  perl -pi -e "s/(memory_mb\s+=\s+)$from/\${1}$to/" "$file"
+}
+
+patch_timeout() {
+  local file="$1" from="$2" to="$3"
+  perl -pi -e "s/(timeout_sec\s+=\s+)$from/\${1}$to/" "$file"
+}
+
+patch_enabled() {
+  local file="$1" val="$2"   # true|false
+  perl -pi -e "s/(enabled\s+=\s+)(true|false)/\${1}$val/" "$file"
+}
+
+# Добавить/заменить блок env_vars в .tf файле
+# Вставляет после строки содержащей "source_dir"
+add_env_var() {
+  local file="$1" key="$2" val="$3"
+  if grep -q "^  env_vars" "$file"; then
+    # уже есть — добавляем строку внутрь блока
+    perl -pi -e "s/(env_vars\s*=\s*\{)/\${1}\n    $key = \"$val\"/" "$file"
+  else
+    # нет — вставляем перед source_dir
+    perl -pi -e "s/(  source_dir)/  env_vars = \{ $key = \"$val\" \}\n\${1}/" "$file"
+  fi
+}
+
+remove_env_vars_block() {
+  local file="$1"
+  perl -0pi -e 's/\s*env_vars\s*=\s*\{[^}]*\}//g' "$file"
+}
+
+# ── Утилиты: terraform output ─────────────────────────────────────────────────
+tf_output() {
+  local chdir="$1" key="$2"
+  (cd "$chdir" && terraform output -raw "$key" 2>/dev/null) || echo ""
+}
+
+# ── Утилиты: счётчик шагов ────────────────────────────────────────────────────
+STEP_NUM=0
+STEP_FAILS=0
+next_step() {
+  STEP_NUM=$((STEP_NUM + 1))
+  step "[$STEP_NUM] $*"
+}
+
+assert_step() {
+  local desc="$1"; shift
+  if "$@"; then
+    ok "$desc"
+    return 0
+  else
+    fail "$desc"
+    STEP_FAILS=$((STEP_FAILS + 1))
+    return 1
+  fi
+}
+
+# ── Чистка артефактов ─────────────────────────────────────────────────────────
+clean_artifacts() {
+  local dir="$1"
+  rm -rf "$dir/.terraform" "$dir/.terraform.lock.hcl" \
+         "$dir/terraform.tfstate" "$dir/terraform.tfstate.backup" \
+         "$dir"/terraform.tfstate.*.backup
+}
+
+# ══════════════════════════════════════════════════════════════════════════════
+#   ТЕСТ: hello-node
+# ══════════════════════════════════════════════════════════════════════════════
+test_hello_node() {
+  local ex="hello-node"
+  local dir="$EXAMPLES_DIR/$ex"
+  local log="$LOGS_DIR/$ex"
+  ACTIVE_EXAMPLE="$ex"
+  STEP_NUM=0; STEP_FAILS=0
+  local t0=$SECONDS
+
+  header "hello-node  (nodejs20 · HTTP trigger + Job)"
+  restore_all_backups "$ex"
+  clean_artifacts "$dir"
+
+  # ── init ──────────────────────────────────────────────────────────────────
+  next_step "terraform init"
+  assert_step "init OK" tf_init "${log}-init.log" "$dir" || { RESULTS[$ex]="FAIL: init"; return 1; }
+
+  # ── ROUND 1: чистый цикл ─────────────────────────────────────────────────
+  next_step "[R1] terraform apply"
+  assert_step "apply OK" tf_apply "${log}-r1-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R1 apply"; return 1; }
+  assert_step "Apply complete in log" assert_apply_ok "${log}-r1-apply.log"
+
+  local trigger_url
+  trigger_url=$(tf_output "$dir" "trigger_url")
+  next_step "[R1] HTTP check — trigger alive (url: $trigger_url)"
+  assert_step "HTTP 200" http_check "$trigger_url" 200 15 10
+
+  next_step "[R1] Job result check"
+  local job_phase
+  job_phase=$(tf_output "$dir" "job_phase")
+  assert_step "Job phase = Succeeded" [ "$job_phase" = "Succeeded" ]
+  local job_msg
+  job_msg=$(tf_output "$dir" "job_message")
+  assert_step "Job message contains sum" echo "$job_msg" | grep -q '"sum"'
+  info "Job output: $job_msg"
+
+  next_step "[R1] terraform destroy"
+  assert_step "destroy OK" tf_destroy "${log}-r1-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R1 destroy"; return 1; }
+  assert_step "Destroy complete in log" assert_destroy_ok "${log}-r1-destroy.log"
+
+  next_step "[R1] HTTP check — trigger gone (want 404/502+unreachable)"
+  # после destroy endpoint должен исчезнуть или вернуть 404
+  local gone=0
+  for i in $(seq 1 18); do
+    local code
+    code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$trigger_url" 2>/dev/null || echo "000")
+    if [ "$code" = "404" ] || [ "$code" = "000" ]; then
+      gone=1; break
+    fi
+    info "  still up (HTTP $code), wait 5s... ($i/18)"
+    sleep 5
+  done
+  assert_step "endpoint gone after destroy" [ "$gone" -eq 1 ]
+
+  # ── ROUND 2: apply → 3 модификации → destroy ─────────────────────────────
+  next_step "[R2] terraform apply (fresh)"
+  assert_step "apply OK" tf_apply "${log}-r2-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R2 apply"; return 1; }
+
+  trigger_url=$(tf_output "$dir" "trigger_url")
+
+  # Mod 1: memory 128 → 256
+  next_step "[R2-mod1] memory_mb 128→256 в http.tf"
+  backup_file "$dir/http.tf"
+  patch_memory "$dir/http.tf" 128 256
+  assert_step "apply mod1 OK" tf_apply "${log}-r2-mod1.log" "$dir"
+  assert_step "mod1: 1 changed" grep -q '1 changed' "${log}-r2-mod1.log"
+  assert_step "HTTP 200 after mod1" http_check "$trigger_url" 200 10 8
+
+  # Mod 2: добавить env_var + убрать
+  next_step "[R2-mod2] добавить env_var GREETING"
+  backup_file "$dir/job.tf"   # на случай если патч затронет и его
+  add_env_var "$dir/http.tf" "GREETING" "stress-test-v1"
+  assert_step "apply mod2 OK" tf_apply "${log}-r2-mod2.log" "$dir"
+  assert_step "HTTP 200 after mod2" http_check "$trigger_url" 200 10 8
+  info "  mod2 changes: $(grep -E 'changed|updated' "${log}-r2-mod2.log" | head -2)"
+
+  # Mod 3: timeout 30 → 45
+  next_step "[R2-mod3] timeout_sec 30→45 в http.tf"
+  patch_timeout "$dir/http.tf" 30 45
+  assert_step "apply mod3 OK" tf_apply "${log}-r2-mod3.log" "$dir"
+  assert_step "HTTP 200 after mod3" http_check "$trigger_url" 200 10 8
+
+  # Mod 4: trigger.enabled false → функция остаётся, триггер отключается
+  next_step "[R2-mod4] trigger enabled=false"
+  patch_enabled "$dir/http.tf" false
+  assert_step "apply mod4 OK" tf_apply "${log}-r2-mod4.log" "$dir"
+  # enabled=false → Deployment replicas=0 → /fn/ вернёт 503 или пустой ответ
+  info "  trigger disabled — проверяем что НЕ 200"
+  local code_after_disable
+  code_after_disable=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$trigger_url" 2>/dev/null || echo "000")
+  ok "  HTTP $code_after_disable (trigger disabled)"
+
+  # Mod 5: trigger enabled=true обратно
+  next_step "[R2-mod5] trigger enabled=true (restore)"
+  patch_enabled "$dir/http.tf" true
+  assert_step "apply mod5 OK" tf_apply "${log}-r2-mod5.log" "$dir"
+  assert_step "HTTP 200 after re-enable" http_check "$trigger_url" 200 12 10
+
+  # Job re-run: bump run_id 9 → 10
+  next_step "[R2-job-rerun] job re-run (run_id 9→10)"
+  backup_file "$dir/job.tf"
+  perl -pi -e 's/(run_id\s+=\s+)9/${1}10/' "$dir/job.tf"
+  assert_step "apply job-rerun OK" tf_apply "${log}-r2-job-rerun.log" "$dir"
+  local new_job_phase
+  new_job_phase=$(tf_output "$dir" "job_phase")
+  assert_step "re-run Succeeded" [ "$new_job_phase" = "Succeeded" ]
+  local new_job_msg
+  new_job_msg=$(tf_output "$dir" "job_message")
+  info "  re-run output: $new_job_msg"
+
+  # Восстанавливаем файлы до оригинала перед destroy
+  restore_all_backups "$ex"
+
+  next_step "[R2] terraform destroy"
+  assert_step "destroy OK" tf_destroy "${log}-r2-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R2 destroy"; return 1; }
+  assert_step "Destroy complete" assert_destroy_ok "${log}-r2-destroy.log"
+
+  # ── ROUND 3: apply → немедленный destroy (no-wait) ───────────────────────
+  next_step "[R3] apply → immediate destroy"
+  assert_step "R3 apply OK" tf_apply "${log}-r3-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R3 apply"; return 1; }
+  assert_step "R3 destroy OK" tf_destroy "${log}-r3-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R3 destroy"; return 1; }
+
+  clean_artifacts "$dir"
+  TIMINGS[$ex]=$((SECONDS - t0))
+
+  if [ "$STEP_FAILS" -eq 0 ]; then
+    RESULTS[$ex]="PASS"
+    ok "hello-node все шаги PASS (${TIMINGS[$ex]}s)"
+  else
+    RESULTS[$ex]="FAIL: $STEP_FAILS step(s) failed"
+    fail "hello-node: $STEP_FAILS шагов с ошибкой"
+  fi
+  ACTIVE_EXAMPLE=""
+}
+
+# ══════════════════════════════════════════════════════════════════════════════
+#   ТЕСТ: simple-node / simple-python  (одинаковая логика, разный runtime)
+# ══════════════════════════════════════════════════════════════════════════════
+test_simple() {
+  local ex="$1"   # simple-node | simple-python
+  local dir="$EXAMPLES_DIR/$ex"
+  local log="$LOGS_DIR/$ex"
+  ACTIVE_EXAMPLE="$ex"
+  STEP_NUM=0; STEP_FAILS=0
+  local t0=$SECONDS
+
+  header "$ex"
+  restore_all_backups "$ex"
+  clean_artifacts "$dir"
+
+  # init
+  next_step "terraform init"
+  assert_step "init OK" tf_init "${log}-init.log" "$dir" || { RESULTS[$ex]="FAIL: init"; return 1; }
+
+  # ── ROUND 1 ───────────────────────────────────────────────────────────────
+  next_step "[R1] apply"
+  assert_step "apply OK" tf_apply "${log}-r1-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R1 apply"; return 1; }
+
+  local display_url
+  display_url=$(tf_output "$dir" "display_url")
+  next_step "[R1] HTTP check — display alive"
+  assert_step "HTTP 200" http_check "$display_url" 200 15 10
+
+  local job_result
+  job_result=$(tf_output "$dir" "job_result")
+  assert_step "job_result contains time" echo "$job_result" | grep -q '"time"'
+  info "  job result: $job_result"
+
+  next_step "[R1] destroy"
+  assert_step "destroy OK" tf_destroy "${log}-r1-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R1 destroy"; return 1; }
+
+  next_step "[R1] endpoint gone check"
+  local gone=0
+  for i in $(seq 1 18); do
+    local code
+    code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 10 "$display_url" 2>/dev/null || echo "000")
+    if [ "$code" = "404" ] || [ "$code" = "000" ]; then gone=1; break; fi
+    info "  still $code, wait 5s... ($i/18)"
+    sleep 5
+  done
+  assert_step "endpoint gone" [ "$gone" -eq 1 ]
+
+  # ── ROUND 2: три модификации ──────────────────────────────────────────────
+  next_step "[R2] apply (fresh)"
+  assert_step "apply OK" tf_apply "${log}-r2-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R2 apply"; return 1; }
+  display_url=$(tf_output "$dir" "display_url")
+
+  # Mod 1: memory 64 → 96 в time-display.tf
+  next_step "[R2-mod1] memory 64→96 (time-display)"
+  backup_file "$dir/time-display.tf"
+  patch_memory "$dir/time-display.tf" 64 96
+  assert_step "apply mod1 OK" tf_apply "${log}-r2-mod1.log" "$dir"
+  assert_step "1 changed" grep -q '1 changed' "${log}-r2-mod1.log"
+  assert_step "HTTP 200 after mod1" http_check "$display_url" 200 12 10
+
+  # Mod 2: memory 64 → 96 в time-getter.tf
+  next_step "[R2-mod2] memory 64→96 (time-getter)"
+  backup_file "$dir/time-getter.tf"
+  patch_memory "$dir/time-getter.tf" 64 96
+  assert_step "apply mod2 OK" tf_apply "${log}-r2-mod2.log" "$dir"
+  assert_step "1 changed" grep -q '1 changed' "${log}-r2-mod2.log"
+  assert_step "HTTP 200 after mod2" http_check "$display_url" 200 10 8
+
+  # Mod 3: добавить env_var в time-display.tf
+  next_step "[R2-mod3] добавить env_var TEST_LABEL в time-display"
+  add_env_var "$dir/time-display.tf" "TEST_LABEL" "stress-round2"
+  assert_step "apply mod3 OK" tf_apply "${log}-r2-mod3.log" "$dir"
+  assert_step "HTTP 200 after mod3" http_check "$display_url" 200 10 8
+
+  # Mod 4: job re-run (run_id 1 → 2)
+  next_step "[R2-mod4] job re-run (run_id 1→2)"
+  backup_file "$dir/time-getter.tf"
+  perl -pi -e 's/(run_id\s+=\s+)1/${1}2/' "$dir/time-getter.tf"
+  assert_step "apply mod4 (job rerun) OK" tf_apply "${log}-r2-mod4.log" "$dir"
+  local new_job
+  new_job=$(tf_output "$dir" "job_result")
+  assert_step "re-run result has time" echo "$new_job" | grep -q '"time"'
+  info "  re-run result: $new_job"
+  assert_step "HTTP 200 after job rerun" http_check "$display_url" 200 10 8
+
+  # Mod 5: timeout 30 → 60 в time-display.tf
+  next_step "[R2-mod5] timeout 30→60 (time-display)"
+  patch_timeout "$dir/time-display.tf" 30 60
+  assert_step "apply mod5 OK" tf_apply "${log}-r2-mod5.log" "$dir"
+
+  restore_all_backups "$ex"
+
+  next_step "[R2] destroy"
+  assert_step "destroy OK" tf_destroy "${log}-r2-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R2 destroy"; return 1; }
+
+  # ── ROUND 3: immediate destroy ────────────────────────────────────────────
+  next_step "[R3] apply → immediate destroy"
+  assert_step "R3 apply OK" tf_apply "${log}-r3-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R3 apply"; return 1; }
+  assert_step "R3 destroy OK" tf_destroy "${log}-r3-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R3 destroy"; return 1; }
+
+  clean_artifacts "$dir"
+  TIMINGS[$ex]=$((SECONDS - t0))
+
+  if [ "$STEP_FAILS" -eq 0 ]; then
+    RESULTS[$ex]="PASS"
+    ok "$ex все шаги PASS (${TIMINGS[$ex]}s)"
+  else
+    RESULTS[$ex]="FAIL: $STEP_FAILS step(s) failed"
+    fail "$ex: $STEP_FAILS шагов с ошибкой"
+  fi
+  ACTIVE_EXAMPLE=""
+}
+
+# ══════════════════════════════════════════════════════════════════════════════
+#   ТЕСТ: notes-python  (postgres + CRUD)
+# ══════════════════════════════════════════════════════════════════════════════
+test_notes_python() {
+  local ex="notes-python"
+  local dir="$EXAMPLES_DIR/$ex"
+  local log="$LOGS_DIR/$ex"
+  ACTIVE_EXAMPLE="$ex"
+  STEP_NUM=0; STEP_FAILS=0
+  local t0=$SECONDS
+
+  header "notes-python  (python3.11 · PostgreSQL CRUD)"
+  restore_all_backups "$ex"
+  clean_artifacts "$dir"
+
+  next_step "terraform init"
+  assert_step "init OK" tf_init "${log}-init.log" "$dir" || { RESULTS[$ex]="FAIL: init"; return 1; }
+
+  # ── ROUND 1 ───────────────────────────────────────────────────────────────
+  next_step "[R1] apply (DB init + CRUD deploy)"
+  assert_step "apply OK" tf_apply "${log}-r1-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R1 apply"; return 1; }
+
+  # Проверяем DB init джобы
+  local tbl_phase idx_phase
+  tbl_phase=$(tf_output "$dir" "db_init_table_status" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('phase','?'))" 2>/dev/null || echo "?")
+  idx_phase=$(tf_output "$dir" "db_init_index_status" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('phase','?'))" 2>/dev/null || echo "?")
+  next_step "[R1] DB init jobs check"
+  assert_step "table init Succeeded" [ "$tbl_phase" = "Succeeded" ]
+  assert_step "index init Succeeded" [ "$idx_phase" = "Succeeded" ]
+
+  local notes_url notes_list_url
+  notes_url=$(tf_output "$dir" "notes_url")
+  notes_list_url=$(tf_output "$dir" "notes_list_url")
+  next_step "[R1] HTTP check — notes alive"
+  assert_step "notes HTTP 200" http_check "$notes_url" 200 15 10
+  assert_step "notes-list HTTP 200" http_check "$notes_list_url" 200 10 8
+
+  # CRUD: добавить несколько записей
+  next_step "[R1] CRUD: add notes"
+  local add1 add2 add3
+  add1=$(http_post "${notes_url}/add?title=StressTest1&body=body1" '{}')
+  add2=$(http_post "${notes_url}/add?title=StressTest2&body=body2" '{}')
+  add3=$(http_post "${notes_url}/add?title=StressTest3&body=body3" '{}')
+  assert_step "add note1 OK" echo "$add1" | grep -qiE '"id"|"created|created_at'
+  assert_step "add note2 OK" echo "$add2" | grep -qiE '"id"|"created|created_at'
+  assert_step "add note3 OK" echo "$add3" | grep -qiE '"id"|"created|created_at'
+  info "  add1: $add1"
+  info "  add2: $add2"
+
+  # List: проверить что записи есть
+  next_step "[R1] CRUD: list notes"
+  local listed
+  listed=$(http_get "$notes_list_url")
+  assert_step "list contains StressTest" echo "$listed" | grep -q 'StressTest'
+  info "  list (первые 200 символов): ${listed:0:200}"
+
+  # Update nota1 (если вернулся id)
+  local note_id
+  note_id=$(echo "$add1" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('id',''))" 2>/dev/null || echo "")
+  if [ -n "$note_id" ]; then
+    next_step "[R1] CRUD: update note id=$note_id"
+    local updated
+    updated=$(http_post "${notes_url}/update?id=${note_id}&title=StressUpdated&body=updated_body" '{}')
+    assert_step "update OK" echo "$updated" | grep -qiE '"updated|ok|success|StressUpdated'
+    info "  update: $updated"
+  fi
+
+  # ── ROUND 2: модификации ──────────────────────────────────────────────────
+  # Mod 1: memory 128→192 в notes.tf
+  next_step "[R2-mod1] memory 128→192 (notes.tf)"
+  backup_file "$dir/notes.tf"
+  patch_memory "$dir/notes.tf" 128 192
+  assert_step "apply mod1 OK" tf_apply "${log}-r2-mod1.log" "$dir"
+  assert_step "1 changed" grep -q '1 changed' "${log}-r2-mod1.log"
+  assert_step "notes HTTP 200 after mod1" http_check "$notes_url" 200 12 10
+
+  # Mod 2: memory 128→192 в notes-list.tf
+  next_step "[R2-mod2] memory 128→192 (notes-list.tf)"
+  backup_file "$dir/notes-list.tf"
+  patch_memory "$dir/notes-list.tf" 128 192
+  assert_step "apply mod2 OK" tf_apply "${log}-r2-mod2.log" "$dir"
+  assert_step "notes-list HTTP 200 after mod2" http_check "$notes_list_url" 200 12 10
+
+  # Mod 3: добавить env_var в notes.tf
+  next_step "[R2-mod3] env_var MAX_NOTES=100 в notes.tf"
+  add_env_var "$dir/notes.tf" "MAX_NOTES" "100"
+  assert_step "apply mod3 OK" tf_apply "${log}-r2-mod3.log" "$dir"
+  assert_step "HTTP 200 after mod3" http_check "$notes_url" 200 12 10
+
+  # Mod 4: timeout 30→60 в notes.tf
+  next_step "[R2-mod4] timeout 30→60 (notes.tf)"
+  patch_timeout "$dir/notes.tf" 30 60
+  assert_step "apply mod4 OK" tf_apply "${log}-r2-mod4.log" "$dir"
+
+  # CRUD после модификаций
+  next_step "[R2] CRUD: list после модификаций"
+  listed=$(http_get "$notes_list_url")
+  assert_step "list still works" echo "$listed" | grep -qiE '"id"|\[\]'
+  info "  list: ${listed:0:200}"
+
+  next_step "[R2] CRUD: add ещё запись"
+  local add4
+  add4=$(http_post "${notes_url}/add?title=PostMod&body=after_modifications" '{}')
+  assert_step "add after mod OK" echo "$add4" | grep -qiE '"id"|"created'
+
+  restore_all_backups "$ex"
+
+  next_step "[R2] destroy"
+  assert_step "destroy OK" tf_destroy "${log}-r2-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R2 destroy"; return 1; }
+
+  # ── ROUND 3: apply → immediate destroy ───────────────────────────────────
+  next_step "[R3] apply → immediate destroy"
+  assert_step "R3 apply OK" tf_apply "${log}-r3-apply.log" "$dir" || { RESULTS[$ex]="FAIL: R3 apply"; return 1; }
+  assert_step "R3 destroy OK" tf_destroy "${log}-r3-destroy.log" "$dir" || { RESULTS[$ex]="FAIL: R3 destroy"; return 1; }
+
+  clean_artifacts "$dir"
+  TIMINGS[$ex]=$((SECONDS - t0))
+
+  if [ "$STEP_FAILS" -eq 0 ]; then
+    RESULTS[$ex]="PASS"
+    ok "notes-python все шаги PASS (${TIMINGS[$ex]}s)"
+  else
+    RESULTS[$ex]="FAIL: $STEP_FAILS step(s) failed"
+    fail "notes-python: $STEP_FAILS шагов с ошибкой"
+  fi
+  ACTIVE_EXAMPLE=""
+}
+
+# ══════════════════════════════════════════════════════════════════════════════
+#   MAIN
+# ══════════════════════════════════════════════════════════════════════════════
+main() {
+  local total_t0=$SECONDS
+
+  echo -e "\n${BOLD}${CYAN}"
+  echo "╔══════════════════════════════════════════════╗"
+  echo "║   sless STRESS TEST   $(date '+%Y-%m-%d %H:%M:%S')    ║"
+  echo "╚══════════════════════════════════════════════╝"
+  echo -e "${RESET}"
+  echo "Примеры : ${EXAMPLES[*]}"
+  echo "Логи    : $LOGS_DIR"
+  echo
+  echo "На каждый пример:"
+  echo "  R1: apply → HTTP-check → destroy → endpoint-gone-check"
+  echo "  R2: apply → 4-5 модификаций (memory, env_var, timeout, re-run) → destroy"
+  echo "  R3: apply → immediate destroy"
+  echo
+
+  for ex in "${EXAMPLES[@]}"; do
+    case "$ex" in
+      hello-node)    test_hello_node   || true ;;
+      simple-node)   test_simple "$ex" || true ;;
+      simple-python) test_simple "$ex" || true ;;
+      notes-python)  test_notes_python || true ;;
+      *) fail "Unknown example: $ex"; RESULTS[$ex]="SKIP: unknown" ;;
+    esac
+  done
+
+  # ── Финальная сводка ──────────────────────────────────────────────────────
+  local total_elapsed=$((SECONDS - total_t0))
+  echo
+  echo -e "${BOLD}${CYAN}════════════════ ИТОГ ════════════════${RESET}"
+  local all_pass=1
+  for ex in "${EXAMPLES[@]}"; do
+    local result="${RESULTS[$ex]:-UNKNOWN}"
+    local timing="${TIMINGS[$ex]:-?}"
+    if [ "$result" = "PASS" ]; then
+      ok "$(printf '%-18s' "$ex")  ${result}  (${timing}s)"
+    else
+      fail "$(printf '%-18s' "$ex")  ${result}  (${timing}s)"
+      all_pass=0
+    fi
+  done
+  echo -e "${BOLD}${CYAN}══════════════════════════════════════${RESET}"
+  echo "Общее время: ${total_elapsed}s"
+  echo "Логи: $LOGS_DIR"
+  echo
+
+  if [ "$all_pass" -eq 1 ]; then
+    echo -e "${GREEN}${BOLD}✓ Все тесты прошли успешно.${RESET}"
+    exit 0
+  else
+    echo -e "${RED}${BOLD}✗ Есть ошибки — см. логи выше и в $LOGS_DIR${RESET}"
+    exit 1
+  fi
+}
+
+main