937 lines
39 KiB
Bash
937 lines
39 KiB
Bash
#!/usr/bin/env bash
|
||
# 2026-03-21 — operator_survival_test.sh
|
||
# ТЕСТ НА ВЫЖИВАНИЕ — суровые многоуровневые нагрузочные сценарии оператора.
|
||
#
|
||
# Группа 5: FLOOD BUILD — 6 функций параллельно (3 Python + 3 Node.js), все до Ready
|
||
# Группа 6: RAPID DISCARD STORM — 30 create→DELETE без build, проверка на мусор
|
||
# Группа 7: CHURN UNDER FIRE — 10 PUT с меняющимися env под 200 параллельными invokes
|
||
# Группа 8: PHOENIX — 3 полных цикла DELETE+recreate одного имени
|
||
# Группа 9: SELF-HEALING MARATHON — 5 раз вручную убить Deployment, ждать восстановления
|
||
# Группа 10: INVOKE HURRICANE — 500 параллельных invokes в 5 волнах, считаем success rate
|
||
#
|
||
# Ожидаемое время прогона: 75-100 минут.
|
||
#
|
||
# ЗАПУСКАТЬ НА VM:
|
||
# nohup bash ~/terra/sless/operator_survival_test.sh > /tmp/survival.log 2>&1 &
|
||
# tail -f /tmp/survival.log
|
||
#
|
||
# Зависимости: curl, python3, zip, kubectl
|
||
|
||
set -uo pipefail
|
||
|
||
# ─── CONFIG ──────────────────────────────────────────────────────────────────
|
||
|
||
TOKEN=$(cat /home/naeel/terra/sless/test.token)
|
||
NS="sless-ffd1f598c169b0ae"
|
||
API="https://sless.kube5s.ru/v1/namespaces/$NS"
|
||
FN_BASE="https://sless.kube5s.ru/fn/$NS"
|
||
DEPLOY_NS="sless-fn-$NS"
|
||
# Уникальный префикс — не пересекается с prod-сервисами и lifecycle-тестом
|
||
TS=$(date +%s)
|
||
SFX="sv-${TS}" # sv = survival
|
||
|
||
# ─── СТАТИСТИКА (глобальная) ──────────────────────────────────────────────────
|
||
|
||
PASS=0; FAIL=0
|
||
declare -A GRP_PASS GRP_FAIL # per-group counters
|
||
|
||
# ─── ЦВЕТА ───────────────────────────────────────────────────────────────────
|
||
|
||
GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'
|
||
CYAN='\033[0;36m'; BOLD='\033[1m'; RESET='\033[0m'
|
||
|
||
# ─── HELPERS ─────────────────────────────────────────────────────────────────
|
||
|
||
_cur_grp="" # текущая группа для per-group счётчиков
|
||
|
||
pass() {
|
||
echo -e " ${GREEN}[PASS]${RESET} $1"
|
||
PASS=$((PASS + 1))
|
||
[[ -n "$_cur_grp" ]] && GRP_PASS[$_cur_grp]=$(( ${GRP_PASS[$_cur_grp]:-0} + 1 ))
|
||
}
|
||
fail() {
|
||
echo -e " ${RED}[FAIL]${RESET} $1"
|
||
FAIL=$((FAIL + 1))
|
||
[[ -n "$_cur_grp" ]] && GRP_FAIL[$_cur_grp]=$(( ${GRP_FAIL[$_cur_grp]:-0} + 1 ))
|
||
}
|
||
info() { echo -e " ${CYAN}[INFO]${RESET} $1"; }
|
||
warn() { echo -e " ${YELLOW}[WARN]${RESET} $1"; }
|
||
section() {
|
||
_cur_grp="G$1"
|
||
echo -e "\n${BOLD}━━━ ГРУППА $1: $2 ━━━${RESET}"
|
||
GRP_PASS[$_cur_grp]=0; GRP_FAIL[$_cur_grp]=0
|
||
}
|
||
|
||
# api_code METHOD URL [body] → HTTP status code
|
||
api_code() {
|
||
local method="$1" url="$2" body="${3:-}"
|
||
local args=(-s -o /dev/null -w "%{http_code}" -m 120
|
||
-H "Authorization: Bearer $TOKEN")
|
||
[[ "$method" != "GET" ]] && args+=(-X "$method")
|
||
[[ -n "$body" ]] && args+=(-H "Content-Type: application/json" -d "$body")
|
||
curl "${args[@]}" "$url"
|
||
}
|
||
|
||
# api_json METHOD URL [body] → response body
|
||
api_json() {
|
||
local method="$1" url="$2" body="${3:-}"
|
||
local args=(-s -m 120 -H "Authorization: Bearer $TOKEN")
|
||
[[ "$method" != "GET" ]] && args+=(-X "$method")
|
||
[[ -n "$body" ]] && args+=(-H "Content-Type: application/json" -d "$body")
|
||
curl "${args[@]}" "$url"
|
||
}
|
||
|
||
check_code() {
|
||
local label="$1" got="$2" want="$3"
|
||
if [[ "$got" == "$want" ]]; then pass "$label → HTTP $got"
|
||
else fail "$label → HTTP $got (ожидали $want)"; fi
|
||
}
|
||
|
||
# svc_phase NAME → phase string
|
||
svc_phase() {
|
||
api_json GET "$API/services/$1" \
|
||
| python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('phase',''))" 2>/dev/null
|
||
}
|
||
|
||
# wait_phase NAME WANT TIMEOUT_SEC → 0=ok 1=timeout 2=Failed
|
||
wait_phase() {
|
||
local name="$1" want="$2" timeout_sec="${3:-300}"
|
||
local deadline=$((SECONDS + timeout_sec))
|
||
while [[ $SECONDS -lt $deadline ]]; do
|
||
local phase
|
||
phase=$(svc_phase "$name")
|
||
[[ "$phase" == "$want" ]] && return 0
|
||
[[ "$phase" == "Failed" && "$want" != "Failed" ]] && return 2
|
||
sleep 5
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# wait_all_ready TIMEOUT NAME1 NAME2 ... → число сколько НЕ достигло Ready
|
||
wait_all_ready() {
|
||
local timeout_sec="$1"; shift
|
||
local names=("$@")
|
||
local deadline=$((SECONDS + timeout_sec))
|
||
local not_ready=( "${names[@]}" )
|
||
while [[ $SECONDS -lt $deadline && ${#not_ready[@]} -gt 0 ]]; do
|
||
local still=()
|
||
for n in "${not_ready[@]}"; do
|
||
local p
|
||
p=$(svc_phase "$n")
|
||
[[ "$p" == "Ready" ]] || still+=("$n")
|
||
done
|
||
not_ready=( "${still[@]}" )
|
||
[[ ${#not_ready[@]} -eq 0 ]] && break
|
||
sleep 5
|
||
done
|
||
echo "${#not_ready[@]}" # количество не дошедших до Ready
|
||
}
|
||
|
||
# make_python_zip NAME → /tmp/surv_py_{NAME}.zip
|
||
# Python echo-handler: возвращает {"ok":true,"env":TEST_VAR,"grp":GRP_VAR}
|
||
make_python_zip() {
|
||
local name="$1"
|
||
local tmpdir; tmpdir=$(mktemp -d)
|
||
cat > "$tmpdir/handler.py" <<'PYEOF'
|
||
import os, json
|
||
|
||
def handle(event):
|
||
return {
|
||
"ok": True,
|
||
"env": os.environ.get("TEST_VAR", ""),
|
||
"grp": os.environ.get("GRP_VAR", ""),
|
||
"msg": event.get("msg", ""),
|
||
"iter": event.get("iter", 0),
|
||
}
|
||
PYEOF
|
||
(cd "$tmpdir" && zip -q "/tmp/surv_py_${name}.zip" handler.py)
|
||
rm -rf "$tmpdir"
|
||
}
|
||
|
||
# make_node_zip NAME → /tmp/surv_node_{NAME}.zip
|
||
make_node_zip() {
|
||
local name="$1"
|
||
local tmpdir; tmpdir=$(mktemp -d)
|
||
cat > "$tmpdir/handler.js" <<'JSEOF'
|
||
'use strict';
|
||
exports.handle = function(event) {
|
||
return {
|
||
ok: true,
|
||
env: process.env.TEST_VAR || '',
|
||
grp: process.env.GRP_VAR || '',
|
||
msg: (event && event.msg) || '',
|
||
iter: (event && event.iter) || 0,
|
||
};
|
||
};
|
||
JSEOF
|
||
(cd "$tmpdir" && zip -q "/tmp/surv_node_${name}.zip" handler.js)
|
||
rm -rf "$tmpdir"
|
||
}
|
||
|
||
# upload_zip NAME ZIPFILE → HTTP code
|
||
upload_zip() {
|
||
local name="$1" zipfile="$2"
|
||
curl -s -o /dev/null -w "%{http_code}" -m 120 \
|
||
-H "Authorization: Bearer $TOKEN" \
|
||
-F "code=@$zipfile" \
|
||
"$API/services/$name/upload"
|
||
}
|
||
|
||
# create_py_deploy NAME MEMORY ENV_VARS_JSON — create + upload python handler
|
||
# return: 0=ok, 1=fail
|
||
create_py_deploy() {
|
||
local name="$1" mem="${2:-128}" envj="${3:-{}}"
|
||
local http
|
||
http=$(api_code POST "$API/services" \
|
||
"{\"name\":\"$name\",\"display_name\":\"survival test\",\
|
||
\"runtime\":\"python3.11\",\"entrypoint\":\"handler.handle\",\
|
||
\"memory_mb\":$mem,\"env_vars\":$envj}")
|
||
[[ "$http" == "201" ]] || { fail "create $name → HTTP $http"; return 1; }
|
||
make_python_zip "$name"
|
||
local ucode
|
||
ucode=$(upload_zip "$name" "/tmp/surv_py_${name}.zip")
|
||
[[ "$ucode" == "200" ]] || { fail "upload $name → HTTP $ucode"; return 1; }
|
||
return 0
|
||
}
|
||
|
||
# create_node_deploy NAME MEMORY ENV_VARS_JSON — create + upload node handler
|
||
create_node_deploy() {
|
||
local name="$1" mem="${2:-128}" envj="${3:-{}}"
|
||
local http
|
||
http=$(api_code POST "$API/services" \
|
||
"{\"name\":\"$name\",\"display_name\":\"survival test\",\
|
||
\"runtime\":\"nodejs20\",\"entrypoint\":\"handler.handle\",\
|
||
\"memory_mb\":$mem,\"env_vars\":$envj}")
|
||
[[ "$http" == "201" ]] || { fail "create $name → HTTP $http"; return 1; }
|
||
make_node_zip "$name"
|
||
local ucode
|
||
ucode=$(upload_zip "$name" "/tmp/surv_node_${name}.zip")
|
||
[[ "$ucode" == "200" ]] || { fail "upload $name → HTTP $ucode"; return 1; }
|
||
return 0
|
||
}
|
||
|
||
# invoke_one NAME → HTTP code
|
||
invoke_one() {
|
||
local name="$1"
|
||
curl -s -o /dev/null -w "%{http_code}" -m 30 \
|
||
-X POST -H "Content-Type: application/json" \
|
||
-d '{"msg":"survival"}' \
|
||
"$FN_BASE/$name"
|
||
}
|
||
|
||
# invoke_burst NAME COUNT PARALLEL — запустить COUNT запросов пачками PARALLEL
|
||
# выводит "SUCCESS=N FAIL=M"
|
||
invoke_burst() {
|
||
local name="$1" total="$2" parallel="$3"
|
||
local results_dir
|
||
results_dir=$(mktemp -d)
|
||
local idx=0
|
||
while [[ $idx -lt $total ]]; do
|
||
local batch_pids=()
|
||
local batch=0
|
||
while [[ $batch -lt $parallel && $idx -lt $total ]]; do
|
||
(
|
||
local code
|
||
code=$(invoke_one "$name")
|
||
echo "$code" > "$results_dir/${idx}"
|
||
) &
|
||
batch_pids+=($!)
|
||
idx=$((idx + 1)); batch=$((batch + 1))
|
||
done
|
||
wait "${batch_pids[@]}"
|
||
done
|
||
local s=0 f=0
|
||
for file in "$results_dir"/*; do
|
||
local c; c=$(cat "$file")
|
||
if [[ "$c" == "200" ]]; then s=$((s + 1)); else f=$((f + 1)); fi
|
||
done
|
||
rm -rf "$results_dir"
|
||
echo "SUCCESS=$s FAIL=$f"
|
||
}
|
||
|
||
# deploy_ns_has NAME → 0 если Deployment существует
|
||
deploy_ns_has() {
|
||
kubectl get deployment -n "$DEPLOY_NS" "$1" --no-headers 2>/dev/null | grep -q "$1"
|
||
}
|
||
|
||
# cleanup NAME — удалить сервис (teardown)
|
||
cleanup() {
|
||
local name="$1"
|
||
[[ -z "$name" ]] && return 0 # пустое имя — пропускаем
|
||
api_code DELETE "$API/services/$name" > /dev/null 2>&1 || true
|
||
sleep 2
|
||
}
|
||
|
||
# Список для teardown — заполняется по ходу тестов
|
||
CLEANUP_NAMES=()
|
||
teardown() {
|
||
if [[ ${#CLEANUP_NAMES[@]} -gt 0 ]]; then
|
||
echo -e "\n${CYAN}[TEARDOWN]${RESET} удаляю тестовые сервисы..."
|
||
for name in "${CLEANUP_NAMES[@]}"; do
|
||
[[ -z "$name" ]] && continue # пропускаем пустые элементы
|
||
cleanup "$name"
|
||
echo -e " ${CYAN}[TEARDOWN]${RESET} удалён: $name"
|
||
done
|
||
fi
|
||
}
|
||
trap teardown EXIT
|
||
|
||
# ─── СТАРТОВЫЙ БАННЕР ────────────────────────────────────────────────────────
|
||
|
||
echo ""
|
||
echo -e "${BOLD}╔══════════════════════════════════════════════════════╗${RESET}"
|
||
echo -e "${BOLD}║ OPERATOR SURVIVAL TEST — sless v0.1.49 ║${RESET}"
|
||
echo -e "${BOLD}╚══════════════════════════════════════════════════════╝${RESET}"
|
||
echo -e " Дата : $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo -e " API : $API"
|
||
echo -e " SFX : $SFX"
|
||
echo ""
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
section 5 "FLOOD BUILD — 6 функций параллельно (3 × Python + 3 × Node.js)"
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
# Цель: убедиться что много одновременных builds не мешают друг другу,
|
||
# все достигают Ready, все отвечают на invoke.
|
||
|
||
FB1="fb1-${SFX}"; FB2="fb2-${SFX}"; FB3="fb3-${SFX}"
|
||
FB4="fb4-${SFX}"; FB5="fb5-${SFX}"; FB6="fb6-${SFX}"
|
||
FB_ALL=("$FB1" "$FB2" "$FB3" "$FB4" "$FB5" "$FB6")
|
||
CLEANUP_NAMES+=( "${FB_ALL[@]}" )
|
||
|
||
info "5.1 CREATE + UPLOAD 6 функций параллельно (3 Python / 3 Node.js)..."
|
||
|
||
(set +e; create_py_deploy "$FB1" 128 '{"GRP_VAR":"py1"}') &
|
||
(set +e; create_py_deploy "$FB2" 192 '{"GRP_VAR":"py2"}') &
|
||
(set +e; create_py_deploy "$FB3" 256 '{"GRP_VAR":"py3"}') &
|
||
(set +e; create_node_deploy "$FB4" 128 '{"GRP_VAR":"nd1"}') &
|
||
(set +e; create_node_deploy "$FB5" 192 '{"GRP_VAR":"nd2"}') &
|
||
(set +e; create_node_deploy "$FB6" 256 '{"GRP_VAR":"nd3"}') &
|
||
wait
|
||
|
||
info "5.2 Проверяем что все 6 CRD созданы (GET → 200)..."
|
||
# Небольшая пауза — контроллер устанавливает phase асинхронно после create
|
||
sleep 5
|
||
all_created=true
|
||
for name in "${FB_ALL[@]}"; do
|
||
http_check=$(api_code GET "$API/services/$name")
|
||
if [[ "$http_check" != "200" ]]; then
|
||
fail "CRD $name не существует → HTTP $http_check (create/upload провалился)"
|
||
all_created=false
|
||
fi
|
||
done
|
||
$all_created && pass "все 6 CRD созданы"
|
||
|
||
info "5.3 Ждём ALL READY (450s для 6 параллельных builds)..."
|
||
not_ready_count=$(wait_all_ready 450 "${FB_ALL[@]}")
|
||
if [[ "$not_ready_count" -eq 0 ]]; then
|
||
pass "все 6 функций достигли Ready ✓"
|
||
else
|
||
fail "$not_ready_count из 6 НЕ достигли Ready за 450s"
|
||
fi
|
||
|
||
# Ждём rollout всех Deployment
|
||
info "5.4 Ждём rollout всех 6 Deployment (120s)..."
|
||
all_rollout=true
|
||
for name in "${FB_ALL[@]}"; do
|
||
if ! kubectl rollout status deployment/"$name" -n "$DEPLOY_NS" --timeout=120s \
|
||
> /dev/null 2>&1; then
|
||
warn "$name rollout не завершился"
|
||
all_rollout=false
|
||
fi
|
||
done
|
||
sleep 5
|
||
$all_rollout && pass "все 6 Deployment rollout завершён"
|
||
|
||
info "5.5 Python-функции: invoke × 3 к каждой, проверяем ok=True..."
|
||
py_ok=true
|
||
for name in "$FB1" "$FB2" "$FB3"; do
|
||
for _i in 1 2 3; do
|
||
rv=$(curl -s -m 20 -X POST -H "Content-Type: application/json" \
|
||
-d '{"msg":"flood"}' "$FN_BASE/$name")
|
||
ok_val=$(echo "$rv" | python3 -c \
|
||
"import sys,json; d=json.load(sys.stdin); print(d.get('ok',''))" 2>/dev/null)
|
||
if [[ "$ok_val" != "True" ]]; then
|
||
fail "$name invoke[$_i] → ok='$ok_val' (resp: ${rv:0:100})"
|
||
py_ok=false; break
|
||
fi
|
||
done
|
||
done
|
||
$py_ok && pass "все Python-функции invoke → ok=True"
|
||
|
||
info "5.6 Node.js-функции: invoke × 3 к каждой, проверяем ok=true..."
|
||
nd_ok=true
|
||
for name in "$FB4" "$FB5" "$FB6"; do
|
||
for _i in 1 2 3; do
|
||
rv=$(curl -s -m 20 -X POST -H "Content-Type: application/json" \
|
||
-d '{"msg":"flood"}' "$FN_BASE/$name")
|
||
ok_val=$(echo "$rv" | python3 -c \
|
||
"import sys,json; d=json.load(sys.stdin); print(str(d.get('ok','')).lower())" 2>/dev/null)
|
||
if [[ "$ok_val" != "true" ]]; then
|
||
fail "$name invoke[$_i] → ok='$ok_val' (resp: ${rv:0:100})"
|
||
nd_ok=false; break
|
||
fi
|
||
done
|
||
done
|
||
$nd_ok && pass "все Node.js-функции invoke → ok=true"
|
||
|
||
info "5.7 GRP_VAR env vars различаются у каждой функции..."
|
||
env_ok=true
|
||
declare -A EXPECTED_GRP
|
||
EXPECTED_GRP["$FB1"]="py1"; EXPECTED_GRP["$FB2"]="py2"; EXPECTED_GRP["$FB3"]="py3"
|
||
EXPECTED_GRP["$FB4"]="nd1"; EXPECTED_GRP["$FB5"]="nd2"; EXPECTED_GRP["$FB6"]="nd3"
|
||
for name in "${FB_ALL[@]}"; do
|
||
rv=$(curl -s -m 20 -X POST -H "Content-Type: application/json" \
|
||
-d '{}' "$FN_BASE/$name")
|
||
got_grp=$(echo "$rv" | python3 -c \
|
||
"import sys,json; d=json.load(sys.stdin); print(d.get('grp',''))" 2>/dev/null)
|
||
want="${EXPECTED_GRP[$name]}"
|
||
if [[ "$got_grp" != "$want" ]]; then
|
||
fail "$name grp='$got_grp' (ожидали '$want')"
|
||
env_ok=false
|
||
fi
|
||
done
|
||
$env_ok && pass "GRP_VAR корректен у всех 6 функций"
|
||
|
||
info "5.8 memory_mb применён в k8s Deployments..."
|
||
mem_ok=true
|
||
declare -A EXPECTED_MEM
|
||
EXPECTED_MEM["$FB1"]="128Mi"; EXPECTED_MEM["$FB2"]="192Mi"; EXPECTED_MEM["$FB3"]="256Mi"
|
||
EXPECTED_MEM["$FB4"]="128Mi"; EXPECTED_MEM["$FB5"]="192Mi"; EXPECTED_MEM["$FB6"]="256Mi"
|
||
for name in "${FB_ALL[@]}"; do
|
||
got_mem=$(kubectl get deployment "$name" -n "$DEPLOY_NS" \
|
||
-o jsonpath='{.spec.template.spec.containers[0].resources.limits.memory}' 2>/dev/null)
|
||
want="${EXPECTED_MEM[$name]}"
|
||
if [[ "$got_mem" != "$want" ]]; then
|
||
fail "$name k8s memory='$got_mem' (ожидали '$want')"
|
||
mem_ok=false
|
||
fi
|
||
done
|
||
$mem_ok && pass "k8s memory limits корректны у всех 6 функций"
|
||
|
||
info "5.9 Удаляем все 6 flood-функций параллельно..."
|
||
del_ok=true
|
||
for name in "${FB_ALL[@]}"; do
|
||
(api_code DELETE "$API/services/$name" > /dev/null 2>&1) &
|
||
done
|
||
wait
|
||
sleep 5
|
||
for name in "${FB_ALL[@]}"; do
|
||
code=$(api_code GET "$API/services/$name")
|
||
[[ "$code" == "404" ]] || { fail "$name: GET после DELETE → $code (ожидали 404)"; del_ok=false; }
|
||
done
|
||
$del_ok && pass "все 6 функций удалены, GET → 404"
|
||
# Убираем из cleanup (уже удалены) — фильтруем пустые и fb-имена
|
||
new_cleanup=()
|
||
for e in "${CLEANUP_NAMES[@]}"; do
|
||
skip=false
|
||
for fn in "${FB_ALL[@]}"; do [[ "$e" == "$fn" ]] && skip=true; done
|
||
[[ -z "$e" ]] && skip=true
|
||
$skip || new_cleanup+=("$e")
|
||
done
|
||
CLEANUP_NAMES=( "${new_cleanup[@]}" )
|
||
|
||
# Ждём очистки k8s
|
||
sleep 15
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
section 6 "RAPID DISCARD STORM — 30 create→DELETE без build"
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
# Цель: убедиться что 30 быстрых create+delete не оставляют orphan CRD.
|
||
# Кейс: пользователь создал функцию, сразу передумал и удалил без upload.
|
||
|
||
info "6.1 Создаём + сразу удаляем 30 функций (10 параллельно, 3 волны)..."
|
||
rd_creates_ok=0
|
||
rd_deletes_ok=0
|
||
rd_total=30
|
||
|
||
results_dir_6=$(mktemp -d)
|
||
overflow=0
|
||
for wave in 1 2 3; do
|
||
wave_pids=()
|
||
for slot in $(seq 1 10); do
|
||
idx=$(( (wave - 1) * 10 + slot ))
|
||
(
|
||
set +e
|
||
rname="rd${idx}-${SFX}"
|
||
# CREATE
|
||
http_c=$(api_code POST "$API/services" \
|
||
"{\"name\":\"$rname\",\"runtime\":\"python3.11\",\
|
||
\"entrypoint\":\"h.h\",\"memory_mb\":128,\"env_vars\":{}}")
|
||
# Немедленно DELETE — не ждём Building
|
||
http_d=$(api_code DELETE "$API/services/$rname")
|
||
echo "${http_c}:${http_d}" > "$results_dir_6/${idx}"
|
||
) &
|
||
wave_pids+=($!)
|
||
done
|
||
wait "${wave_pids[@]}"
|
||
done
|
||
|
||
# Подсчёт результатов
|
||
for file in "$results_dir_6"/*; do
|
||
pair=$(cat "$file")
|
||
c_code="${pair%%:*}"
|
||
d_code="${pair##*:}"
|
||
[[ "$c_code" == "201" ]] && rd_creates_ok=$((rd_creates_ok + 1))
|
||
[[ "$d_code" == "204" || "$d_code" == "404" ]] && rd_deletes_ok=$((rd_deletes_ok + 1))
|
||
done
|
||
rm -rf "$results_dir_6"
|
||
|
||
if [[ $rd_creates_ok -ge 27 ]]; then # допустим ≤3 ошибки (гонка)
|
||
pass "$rd_creates_ok/$rd_total create вернули 201"
|
||
else
|
||
fail "$rd_creates_ok/$rd_total create вернули 201 (ожидали ≥27)"
|
||
fi
|
||
if [[ $rd_deletes_ok -ge 27 ]]; then
|
||
pass "$rd_deletes_ok/$rd_total delete вернули 204/404"
|
||
else
|
||
fail "$rd_deletes_ok/$rd_total delete ок (ожидали ≥27)"
|
||
fi
|
||
|
||
info "6.2 Ждём 20s чтобы finalizer-ы отработали..."
|
||
sleep 20
|
||
|
||
info "6.3 Проверяем что нет orphan CRD в namespace $NS..."
|
||
orphan_count=$(api_json GET "$API/services" \
|
||
| python3 -c "
|
||
import sys, json
|
||
data = json.load(sys.stdin)
|
||
items = data if isinstance(data, list) else data.get('items', data.get('services', []))
|
||
surv = [i.get('name','') for i in items if '${SFX}' in i.get('name','')]
|
||
print(len(surv))
|
||
print('\n'.join(surv))
|
||
" 2>/dev/null | head -1)
|
||
if [[ "$orphan_count" -eq 0 ]]; then
|
||
pass "нет orphan CRD — все $rd_total функций удалены чисто"
|
||
else
|
||
fail "$orphan_count orphan CRD обнаружено после rapid discard"
|
||
fi
|
||
|
||
info "6.4 Нет orphan Deployment/Service в deploy namespace $DEPLOY_NS..."
|
||
orphan_k8s=$(kubectl get deployment -n "$DEPLOY_NS" --no-headers 2>/dev/null \
|
||
| grep "${SFX}" | wc -l)
|
||
if [[ "$orphan_k8s" -eq 0 ]]; then
|
||
pass "нет orphan Deployment в $DEPLOY_NS"
|
||
else
|
||
fail "$orphan_k8s orphan Deployment в $DEPLOY_NS после rapid discard"
|
||
fi
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
section 7 "CHURN UNDER FIRE — 10 PUT env-updates под 200 параллельными invokes"
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
# Цель: обновления конфигурации не ломают работу функции под нагрузкой.
|
||
# Метрика: success rate invoke >= 90%, финальный env корректный.
|
||
|
||
CU="churn-${SFX}"
|
||
CLEANUP_NAMES+=("$CU")
|
||
|
||
info "7.1 Создаём и деплоим churn-функцию..."
|
||
if create_py_deploy "$CU" 192 '{"TEST_VAR":"ver-0","GRP_VAR":"churn"}'; then
|
||
pass "create+upload $CU OK"
|
||
else
|
||
fail "create+upload $CU провалился, пропускаем группу 7"
|
||
# Очистка + переход к следующей группе
|
||
cleanup "$CU"
|
||
fi
|
||
|
||
info "7.2 Ждём Ready (300s)..."
|
||
cu_ready=0
|
||
wait_phase "$CU" "Ready" 300 || cu_ready=$?
|
||
if [[ $cu_ready -eq 0 ]]; then
|
||
pass "$CU Ready"
|
||
kubectl rollout status deployment/"$CU" -n "$DEPLOY_NS" --timeout=60s \
|
||
> /dev/null 2>&1 || true
|
||
sleep 3
|
||
else
|
||
fail "$CU не вышел в Ready (фаза: $(svc_phase "$CU")), пропускаем тест"
|
||
cleanup "$CU"; CLEANUP_NAMES=( "${CLEANUP_NAMES[@]/$CU}" )
|
||
# jump over group 7 остальные тесты пропустим
|
||
cu_ready=99
|
||
fi
|
||
|
||
if [[ $cu_ready -eq 0 ]]; then
|
||
info "7.3 Запускаем фоновый поток: 200 invoke (10 параллельно × 20 волн)..."
|
||
# Invoke в фоне — пишем коды в файлы
|
||
churn_results=$(mktemp -d)
|
||
(
|
||
for wave in $(seq 1 20); do
|
||
wave_pids=()
|
||
for _b in $(seq 1 10); do
|
||
idx=$(( (wave - 1) * 10 + _b ))
|
||
(
|
||
code=$(invoke_one "$CU")
|
||
echo "$code" > "$churn_results/${idx}"
|
||
) &
|
||
wave_pids+=($!)
|
||
done
|
||
wait "${wave_pids[@]}"
|
||
sleep 0.5 # небольшая пауза между волнами чтобы не throttle
|
||
done
|
||
) &
|
||
fire_pid=$!
|
||
|
||
info "7.4 Пока идут invoke — делаем 10 PUT с нарастающим TEST_VAR..."
|
||
for ver in $(seq 1 10); do
|
||
sleep 4 # ~каждые 4s новый PUT
|
||
http_put=$(api_code PUT "$API/services/$CU" \
|
||
"{\"runtime\":\"python3.11\",\"entrypoint\":\"handler.handle\",\
|
||
\"memory_mb\":192,\"env_vars\":{\"TEST_VAR\":\"ver-${ver}\",\"GRP_VAR\":\"churn\"}}")
|
||
if [[ "$http_put" == "200" ]]; then
|
||
info " PUT ver-${ver} → 200 ✓"
|
||
else
|
||
fail "PUT ver-${ver} → HTTP $http_put"
|
||
fi
|
||
done
|
||
|
||
info "7.5 Ждём завершения фонового invoke-потока..."
|
||
wait $fire_pid
|
||
|
||
# Подсчёт invoke результатов
|
||
cu_success=0; cu_fail_cnt=0
|
||
for file in "$churn_results"/*; do
|
||
code=$(cat "$file")
|
||
if [[ "$code" == "200" ]]; then cu_success=$((cu_success + 1))
|
||
else cu_fail_cnt=$((cu_fail_cnt + 1)); fi
|
||
done
|
||
rm -rf "$churn_results"
|
||
total_invokes=$((cu_success + cu_fail_cnt))
|
||
success_pct=0
|
||
[[ $total_invokes -gt 0 ]] && success_pct=$(( cu_success * 100 / total_invokes ))
|
||
info " Invoke сводка: $cu_success/$total_invokes успешных ($success_pct%)"
|
||
|
||
if [[ $success_pct -ge 90 ]]; then
|
||
pass "invoke success rate $success_pct% ≥ 90% под 10 rolling updates"
|
||
else
|
||
fail "invoke success rate $success_pct% < 90% (много гонок при update?)"
|
||
fi
|
||
|
||
info "7.6 Ждём rollout финального обновления (ver-10)..."
|
||
sleep 10
|
||
kubectl rollout status deployment/"$CU" -n "$DEPLOY_NS" --timeout=90s \
|
||
> /dev/null 2>&1 || true
|
||
sleep 5
|
||
|
||
info "7.7 Финальный invoke → TEST_VAR должен быть ver-10..."
|
||
final_resp=$(curl -s -m 20 -X POST -H "Content-Type: application/json" \
|
||
-d '{}' "$FN_BASE/$CU")
|
||
final_env=$(echo "$final_resp" | python3 -c \
|
||
"import sys,json; d=json.load(sys.stdin); print(d.get('env',''))" 2>/dev/null)
|
||
if [[ "$final_env" == "ver-10" ]]; then
|
||
pass "финальный env=ver-10 ✓"
|
||
else
|
||
fail "финальный env='$final_env' (ожидали 'ver-10')"
|
||
fi
|
||
|
||
info "7.8 k8s Deployment env = ver-10..."
|
||
k8s_env_cu=$(kubectl get deployment "$CU" -n "$DEPLOY_NS" -o json 2>/dev/null \
|
||
| python3 -c "import sys,json
|
||
d=json.load(sys.stdin)
|
||
envs=d['spec']['template']['spec']['containers'][0].get('env',[])
|
||
val=next((e['value'] for e in envs if e['name']=='TEST_VAR'), '')
|
||
print(val)" 2>/dev/null)
|
||
if [[ "$k8s_env_cu" == "ver-10" ]]; then
|
||
pass "k8s Deployment TEST_VAR=ver-10 ✓"
|
||
else
|
||
fail "k8s Deployment TEST_VAR='$k8s_env_cu' (ожидали 'ver-10')"
|
||
fi
|
||
fi
|
||
|
||
info "7.9 Удаляем churn-функцию..."
|
||
check_code "DELETE $CU" "$(api_code DELETE "$API/services/$CU")" "204"
|
||
CLEANUP_NAMES=( "${CLEANUP_NAMES[@]/$CU}" )
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
section 8 "PHOENIX — 3 цикла DELETE + recreate одного имени"
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
# Цель: после DELETE сервиса его имя можно переиспользовать немедленно.
|
||
# Финалайзер должен корректно удалить k8s ресурсы прежде чем CRD исчезнет.
|
||
# Каждый цикл: create → wait Ready → DELETE → wait 404 → снова create.
|
||
|
||
PX="phoenix-${SFX}"
|
||
CLEANUP_NAMES+=("$PX")
|
||
|
||
phoenix_all_ok=true
|
||
for cycle in 1 2 3; do
|
||
info "8.${cycle} PHOENIX цикл $cycle/3..."
|
||
|
||
# CREATE
|
||
cx_code=$(api_code POST "$API/services" \
|
||
"{\"name\":\"$PX\",\"display_name\":\"phoenix c$cycle\",\
|
||
\"runtime\":\"python3.11\",\"entrypoint\":\"handler.handle\",\
|
||
\"memory_mb\":128,\"env_vars\":{\"TEST_VAR\":\"cycle-${cycle}\"}}")
|
||
if [[ "$cx_code" != "201" ]]; then
|
||
fail " phoenix цикл $cycle: CREATE → HTTP $cx_code (ожидали 201)"
|
||
phoenix_all_ok=false; continue
|
||
fi
|
||
|
||
# UPLOAD
|
||
make_python_zip "${PX}_cycle${cycle}"
|
||
cp "/tmp/surv_py_${PX}_cycle${cycle}.zip" "/tmp/surv_py_${PX}.zip" 2>/dev/null || true
|
||
make_python_zip "${PX}"
|
||
u_code=$(upload_zip "$PX" "/tmp/surv_py_${PX}.zip")
|
||
if [[ "$u_code" != "200" ]]; then
|
||
fail " phoenix цикл $cycle: UPLOAD → HTTP $u_code"
|
||
phoenix_all_ok=false; cleanup "$PX"; continue
|
||
fi
|
||
|
||
# WAIT READY
|
||
px_w=0
|
||
wait_phase "$PX" "Ready" 300 || px_w=$?
|
||
if [[ $px_w -ne 0 ]]; then
|
||
fail " phoenix цикл $cycle: не дошёл до Ready (w=$px_w, phase=$(svc_phase "$PX"))"
|
||
phoenix_all_ok=false; cleanup "$PX"; continue
|
||
fi
|
||
kubectl rollout status deployment/"$PX" -n "$DEPLOY_NS" --timeout=60s \
|
||
> /dev/null 2>&1 || true
|
||
sleep 3
|
||
|
||
# INVOKE — убедиться что работает и env правильный
|
||
px_resp=$(curl -s -m 20 -X POST -H "Content-Type: application/json" \
|
||
-d '{}' "$FN_BASE/$PX")
|
||
px_env=$(echo "$px_resp" | python3 -c \
|
||
"import sys,json; d=json.load(sys.stdin); print(d.get('env',''))" 2>/dev/null)
|
||
if [[ "$px_env" == "cycle-${cycle}" ]]; then
|
||
pass " phoenix цикл $cycle: Ready + invoke env=cycle-${cycle} ✓"
|
||
else
|
||
fail " phoenix цикл $cycle: invoke env='$px_env' (ожидали 'cycle-${cycle}')"
|
||
phoenix_all_ok=false
|
||
fi
|
||
|
||
# DELETE
|
||
del_code=$(api_code DELETE "$API/services/$PX")
|
||
if [[ "$del_code" != "204" ]]; then
|
||
fail " phoenix цикл $cycle: DELETE → HTTP $del_code"
|
||
phoenix_all_ok=false; continue
|
||
fi
|
||
|
||
# WAIT 404 — ждём когда finalizer уберёт CRD
|
||
info " Ждём когда CRD исчезнет (finalizer, 60s)..."
|
||
gone=false
|
||
for _w in $(seq 1 12); do
|
||
sleep 5
|
||
code404=$(api_code GET "$API/services/$PX")
|
||
if [[ "$code404" == "404" ]]; then gone=true; break; fi
|
||
done
|
||
if ! $gone; then
|
||
fail " phoenix цикл $cycle: CRD $PX не исчез за 60s после DELETE"
|
||
phoenix_all_ok=false
|
||
fi
|
||
|
||
# Проверяем k8s Deployment тоже удалён
|
||
k8s_gone=false
|
||
for _w in $(seq 1 6); do
|
||
sleep 5
|
||
deploy_ns_has "$PX" || { k8s_gone=true; break; }
|
||
done
|
||
if ! $k8s_gone; then
|
||
fail " phoenix цикл $cycle: k8s Deployment не удалён за 30s"
|
||
phoenix_all_ok=false
|
||
fi
|
||
|
||
info " Цикл $cycle завершён — имя $PX освободилось"
|
||
done
|
||
|
||
if $phoenix_all_ok; then
|
||
pass "PHOENIX: все 3 цикла DELETE+recreate прошли успешно"
|
||
fi
|
||
CLEANUP_NAMES=( "${CLEANUP_NAMES[@]/$PX}" )
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
section 9 "SELF-HEALING MARATHON — 5 раз вручную убить Deployment"
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
# Цель: оператор с RequeueAfter:60s восстанавливает Deployment каждый раз.
|
||
# Без сбоев подряд: удаляем → ждём 90s → убедиться что пересоздан → invoke OK.
|
||
|
||
SH="sheal-${SFX}"
|
||
CLEANUP_NAMES+=("$SH")
|
||
|
||
info "9.1 Создаём и деплоим self-healing функцию..."
|
||
if create_py_deploy "$SH" 128 '{"TEST_VAR":"sheal"}'; then
|
||
sh_ready=0
|
||
wait_phase "$SH" "Ready" 300 || sh_ready=$?
|
||
if [[ $sh_ready -eq 0 ]]; then
|
||
pass "$SH Ready перед self-healing marathon"
|
||
kubectl rollout status deployment/"$SH" -n "$DEPLOY_NS" --timeout=60s \
|
||
> /dev/null 2>&1 || true
|
||
sleep 3
|
||
else
|
||
fail "$SH не вышел в Ready, пропускаем группу 9"
|
||
sh_ready=99
|
||
fi
|
||
else
|
||
fail "create/upload $SH провалился, пропускаем группу 9"
|
||
sh_ready=99
|
||
fi
|
||
|
||
if [[ "${sh_ready:-0}" -eq 0 ]]; then
|
||
sheal_ok=0
|
||
for kill_round in 1 2 3 4 5; do
|
||
info "9.kill$kill_round — убиваем Deployment (раунд $kill_round/5)..."
|
||
kubectl delete deployment "$SH" -n "$DEPLOY_NS" > /dev/null 2>&1
|
||
|
||
info " Ждём пересоздания (90s = минимум один RequeueAfter: 60s цикл + delta)..."
|
||
recreated=false
|
||
for _w in $(seq 1 18); do
|
||
sleep 5
|
||
if deploy_ns_has "$SH"; then recreated=true; break; fi
|
||
done
|
||
|
||
if $recreated; then
|
||
# Ждём pod готовности
|
||
kubectl rollout status deployment/"$SH" -n "$DEPLOY_NS" --timeout=90s \
|
||
> /dev/null 2>&1 || true
|
||
sleep 3
|
||
# Invoke должен работать
|
||
sh_resp=$(curl -s -m 30 -X POST -H "Content-Type: application/json" \
|
||
-d '{"iter":'"$kill_round"'}' "$FN_BASE/$SH")
|
||
sh_ok=$(echo "$sh_resp" | python3 -c \
|
||
"import sys,json; d=json.load(sys.stdin); print(d.get('ok',''))" 2>/dev/null)
|
||
if [[ "$sh_ok" == "True" ]]; then
|
||
pass " раунд $kill_round: Deployment пересоздан + invoke OK"
|
||
sheal_ok=$((sheal_ok + 1))
|
||
else
|
||
fail " раунд $kill_round: Deployment пересоздан, но invoke → ${sh_resp:0:80}"
|
||
fi
|
||
else
|
||
fail " раунд $kill_round: Deployment НЕ пересоздан за 90s после удаления"
|
||
fi
|
||
done
|
||
info "Self-healing итог: $sheal_ok/5 раундов прошли"
|
||
fi
|
||
|
||
info "9.final Удаляем $SH..."
|
||
check_code "DELETE $SH" "$(api_code DELETE "$API/services/$SH")" "204"
|
||
CLEANUP_NAMES=( "${CLEANUP_NAMES[@]/$SH}" )
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
section 10 "INVOKE HURRICANE — 500 параллельных invokes в 5 волнах"
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
# Цель: функция держит нагрузку 100 одновременных запросов × 5 волн = 500 total.
|
||
# Метрика: success rate >= 90%, нет зависаний.
|
||
|
||
HU="hurr-${SFX}"
|
||
CLEANUP_NAMES+=("$HU")
|
||
|
||
info "10.1 Создаём hurricane-функцию (Node.js, 256Mi)..."
|
||
if create_node_deploy "$HU" 256 '{"TEST_VAR":"hurricane","GRP_VAR":"load"}'; then
|
||
hu_ready=0
|
||
wait_phase "$HU" "Ready" 300 || hu_ready=$?
|
||
if [[ $hu_ready -eq 0 ]]; then
|
||
pass "$HU Ready"
|
||
kubectl rollout status deployment/"$HU" -n "$DEPLOY_NS" --timeout=90s \
|
||
> /dev/null 2>&1 || true
|
||
sleep 5
|
||
else
|
||
fail "$HU не вышел в Ready, пропускаем группу 10"
|
||
hu_ready=99
|
||
fi
|
||
else
|
||
fail "create/upload $HU провалился, пропускаем группу 10"
|
||
hu_ready=99
|
||
fi
|
||
|
||
if [[ "${hu_ready:-0}" -eq 0 ]]; then
|
||
info "10.2 Прогрев: 10 последовательных invoke перед нагрузкой..."
|
||
warmup_ok=0
|
||
for _w in $(seq 1 10); do
|
||
code=$(invoke_one "$HU")
|
||
[[ "$code" == "200" ]] && warmup_ok=$((warmup_ok + 1))
|
||
done
|
||
if [[ $warmup_ok -ge 8 ]]; then
|
||
pass "прогрев: $warmup_ok/10 OK"
|
||
else
|
||
fail "прогрев: $warmup_ok/10 OK (функция не стабильна)"
|
||
fi
|
||
|
||
info "10.3 ВОЛНА 1-5: 100 параллельных запросов в каждой волне (500 total)..."
|
||
total_ok=0; total_fail=0
|
||
for wave in 1 2 3 4 5; do
|
||
info " ВОЛНА $wave/5 (100 параллельных)..."
|
||
# Запускаем 100 параллельно
|
||
wave_dir=$(mktemp -d)
|
||
wave_pids=()
|
||
for req in $(seq 1 100); do
|
||
(
|
||
code=$(invoke_one "$HU")
|
||
echo "$code" > "$wave_dir/${req}"
|
||
) &
|
||
wave_pids+=($!)
|
||
done
|
||
wait "${wave_pids[@]}"
|
||
|
||
wave_ok=0; wave_fail=0
|
||
for file in "$wave_dir"/*; do
|
||
c=$(cat "$file")
|
||
if [[ "$c" == "200" ]]; then wave_ok=$((wave_ok + 1))
|
||
else wave_fail=$((wave_fail + 1)); fi
|
||
done
|
||
rm -rf "$wave_dir"
|
||
total_ok=$((total_ok + wave_ok))
|
||
total_fail=$((total_fail + wave_fail))
|
||
info " волна $wave: $wave_ok/100 OK, $wave_fail FAIL"
|
||
sleep 2 # небольшой перерыв между волнами
|
||
done
|
||
|
||
total=$((total_ok + total_fail))
|
||
pct=0
|
||
[[ $total -gt 0 ]] && pct=$(( total_ok * 100 / total ))
|
||
info " Итого: $total_ok/$total OK ($pct%)"
|
||
|
||
if [[ $pct -ge 90 ]]; then
|
||
pass "HURRICANE: $total_ok/$total invoke → ${pct}% success rate (≥90%) ✓"
|
||
else
|
||
fail "HURRICANE: $total_ok/$total invoke → ${pct}% success rate (<90%)"
|
||
fi
|
||
|
||
info "10.4 Функция стабильна после шторма — финальный invoke..."
|
||
sleep 5
|
||
final_code=$(invoke_one "$HU")
|
||
if [[ "$final_code" == "200" ]]; then
|
||
pass "финальный invoke после нагрузки → 200 ✓"
|
||
else
|
||
fail "финальный invoke после нагрузки → $final_code"
|
||
fi
|
||
fi
|
||
|
||
info "10.5 Удаляем hurricane-функцию..."
|
||
check_code "DELETE $HU" "$(api_code DELETE "$API/services/$HU")" "204"
|
||
CLEANUP_NAMES=( "${CLEANUP_NAMES[@]/$HU}" )
|
||
|
||
# ═══════════════════════════════════════════════════════════════════════════════
|
||
echo ""
|
||
echo -e "${BOLD}╔══════════════════════════════════════════════════════╗${RESET}"
|
||
echo -e "${BOLD}║ ИТОГИ SURVIVAL ║${RESET}"
|
||
echo -e "${BOLD}╚══════════════════════════════════════════════════════╝${RESET}"
|
||
echo ""
|
||
echo " Время завершения : $(date '+%Y-%m-%d %H:%M:%S')"
|
||
echo ""
|
||
echo " Всего тестов : $((PASS + FAIL))"
|
||
echo " PASS : $PASS"
|
||
echo " FAIL : $FAIL"
|
||
echo ""
|
||
|
||
# Per-group итоги
|
||
for grp in G5 G6 G7 G8 G9 G10; do
|
||
g_pass=${GRP_PASS[$grp]:-0}
|
||
g_fail=${GRP_FAIL[$grp]:-0}
|
||
g_total=$((g_pass + g_fail))
|
||
label=""
|
||
case "$grp" in
|
||
G5) label="FLOOD BUILD";; G6) label="RAPID DISCARD STORM";;
|
||
G7) label="CHURN UNDER FIRE";; G8) label="PHOENIX";;
|
||
G9) label="SELF-HEALING MARATHON";; G10) label="INVOKE HURRICANE";;
|
||
esac
|
||
if [[ $g_fail -eq 0 ]]; then
|
||
echo -e " ${GREEN}✓${RESET} Группа $grp ($label): ${g_pass}/${g_total}"
|
||
else
|
||
echo -e " ${RED}✗${RESET} Группа $grp ($label): ${g_pass}/${g_total} (FAIL: $g_fail)"
|
||
fi
|
||
done
|
||
|
||
echo ""
|
||
if [[ $FAIL -eq 0 ]]; then
|
||
echo -e " ${GREEN}${BOLD}✓ ВЫЖИЛ — ВСЕ ТЕСТЫ ПРОШЛИ${RESET}"
|
||
else
|
||
echo -e " ${RED}${BOLD}✗ ПРОВАЛ — $FAIL тестов упало${RESET}"
|
||
fi
|
||
echo ""
|