sless/test_harbor_load.sh

#!/usr/bin/env bash
# 2026-04-09
# Нагрузочный тест Harbor: параллельные curl-запросы, замер латентности,
# фиксация зависаний по таймауту. Позволяет воспроизвести и измерить
# intermittent-проблему с HTTP-протоколом на конкретном хосте.

set -uo pipefail

# ── Конфигурация ─────────────────────────────────────────────────────────────
SECRETS_FILE="${SECRETS_FILE:-secrets/pearlharbor_registry.txt}"
DURATION=${DURATION:-60}        # секунд
CONCURRENCY=${CONCURRENCY:-10}  # параллельных воркеров
TIMEOUT=${TIMEOUT:-8}           # таймаут одного запроса (сек)
LOGDIR="${LOGDIR:-/tmp/harbor_load_$(date +%Y%m%d_%H%M%S)}"

# ── Читаем секреты ────────────────────────────────────────────────────────────
if [ ! -f "$SECRETS_FILE" ]; then
  echo "ERROR: secrets file not found: $SECRETS_FILE" >&2
  exit 1
fi
connection_url=$(grep -E '^connection_url=' "$SECRETS_FILE" | cut -d'=' -f2-)
admin_pass=$(grep -E '^admin_pass='      "$SECRETS_FILE" | cut -d'=' -f2-)
REGISTRY=$(echo "$connection_url" | sed -E 's~https?://~~' | sed -E 's~/$~~')
BASE_URL="https://$REGISTRY"

# ── Точки нагрузки (URL + принудительный протокол) ───────────────────────────
# Цель: нагрузить Harbor разными типами запросов и обоими протоколами,
# чтобы поймать и зафиксировать intermittent-зависания.
declare -a TARGETS=(
  "h2   $BASE_URL/api/v2.0/ping"
  "h1   $BASE_URL/api/v2.0/ping"
  "h2   $BASE_URL/api/v2.0/projects"
  "h1   $BASE_URL/api/v2.0/projects"
  "h2   $BASE_URL/v2/"
  "h1   $BASE_URL/v2/"
)

mkdir -p "$LOGDIR"
RESULT_LOG="$LOGDIR/results.csv"
echo "ts,worker,proto,url,http_code,time_total,timeout" > "$RESULT_LOG"

echo "=== Harbor Load Test ==="
echo "Registry : $BASE_URL"
echo "Duration : ${DURATION}s"
echo "Workers  : $CONCURRENCY"
echo "Timeout  : ${TIMEOUT}s per request"
echo "Log      : $LOGDIR"
echo ""

END_TIME=$(( $(date +%s) + DURATION ))

# ── Один воркер ──────────────────────────────────────────────────────────────
harbor_worker() {
  local wid="$1"
  local target_idx=$(( wid % ${#TARGETS[@]} ))
  local entry="${TARGETS[$target_idx]}"
  local proto=$(echo "$entry" | awk '{print $1}')
  local url=$(echo   "$entry" | awk '{print $2}')

  local proto_flag=""
  if [ "$proto" = "h1" ]; then
    proto_flag="--http1.1"
  else
    proto_flag="--http2"
  fi

  while [ "$(date +%s)" -lt "$END_TIME" ]; do
    local ts
    ts=$(date +%s%3N)
    # Замеряем время ответа и HTTP-код
    local result
    result=$(curl -sk $proto_flag \
      --max-time "$TIMEOUT" \
      -u "admin:$admin_pass" \
      -o /dev/null \
      -w "%{http_code} %{time_total}" \
      "$url" 2>/dev/null) || result="000 $TIMEOUT"

    local http_code time_total timed_out
    http_code=$(echo "$result"  | awk '{print $1}')
    time_total=$(echo "$result" | awk '{print $2}')

    # Считаем тайм-аут если http_code=000 или время близко к лимиту
    if [ "$http_code" = "000" ]; then
      timed_out=1
    else
      timed_out=0
    fi

    echo "$ts,$wid,$proto,$url,$http_code,$time_total,$timed_out" >> "$RESULT_LOG"
  done
}

export -f harbor_worker
export END_TIME TIMEOUT LOGDIR RESULT_LOG admin_pass
export -a TARGETS

# ── Запускаем воркеров параллельно ───────────────────────────────────────────
echo "Starting $CONCURRENCY workers for ${DURATION}s..."
PIDS=()
for i in $(seq 0 $(( CONCURRENCY - 1 ))); do
  harbor_worker "$i" &
  PIDS+=($!)
done

# ── Прогресс каждые 5 секунд ─────────────────────────────────────────────────
while [ "$(date +%s)" -lt "$END_TIME" ]; do
  sleep 5
  local_count=$(wc -l < "$RESULT_LOG")
  local_timeouts=$(grep -c ",1$" "$RESULT_LOG" 2>/dev/null || echo 0)
  elapsed=$(( DURATION - ( END_TIME - $(date +%s) ) ))
  echo "[+${elapsed}s] requests: $(( local_count - 1 )), timeouts: $local_timeouts"
done

# ── Ждём завершения воркеров ─────────────────────────────────────────────────
for pid in "${PIDS[@]}"; do
  wait "$pid" 2>/dev/null || true
done

echo ""
echo "=== Results ==="

# ── Статистика через python3 ─────────────────────────────────────────────────
python3 - "$RESULT_LOG" << 'PYEOF'
import sys, csv
from collections import defaultdict

log_file = sys.argv[1]
rows = []
with open(log_file) as f:
    reader = csv.DictReader(f)
    for r in reader:
        rows.append(r)

total = len(rows)
timeouts = sum(1 for r in rows if r['timeout'] == '1')
success  = sum(1 for r in rows if r['timeout'] == '0' and r['http_code'].startswith('2'))

times = [float(r['time_total']) for r in rows if r['timeout'] == '0']
times.sort()

def pct(lst, p):
    if not lst: return 0
    idx = int(len(lst) * p / 100)
    return lst[min(idx, len(lst)-1)]

print(f"Total requests : {total}")
print(f"Success (2xx)  : {success} ({100*success//total if total else 0}%)")
print(f"Timeouts       : {timeouts} ({100*timeouts//total if total else 0}%)")
other = total - success - timeouts
print(f"Other errors   : {other}")
if times:
    print(f"Latency (ok)   : min={min(times):.3f}s  median={pct(times,50):.3f}s  p95={pct(times,95):.3f}s  max={max(times):.3f}s")

print()
print("--- By protocol ---")
by_proto = defaultdict(lambda: {'ok':0,'fail':0,'times':[]})
for r in rows:
    p = r['proto']
    if r['timeout'] == '1':
        by_proto[p]['fail'] += 1
    elif r['http_code'].startswith('2'):
        by_proto[p]['ok'] += 1
        by_proto[p]['times'].append(float(r['time_total']))
    else:
        by_proto[p]['fail'] += 1

for proto, d in sorted(by_proto.items()):
    t = sorted(d['times'])
    p95 = pct(t, 95) if t else 0
    print(f"  {proto}: ok={d['ok']} fail={d['fail']} p95={p95:.3f}s")

print()
print("--- By URL ---")
by_url = defaultdict(lambda: {'ok':0,'to':0})
for r in rows:
    u = r['url']
    if r['timeout'] == '1':
        by_url[u]['to'] += 1
    elif r['http_code'].startswith('2'):
        by_url[u]['ok'] += 1
    # else: counted as non-2xx

for url, d in sorted(by_url.items(), key=lambda x: x[0]):
    print(f"  {url}: ok={d['ok']} timeout={d['to']}")
PYEOF

echo ""
echo "Raw log: $RESULT_LOG"