tests: compare_test v2 — 🔴BUG🟡LAG🟢BETTER, _norm(), _cmp_val(), valueList

This commit is contained in:
2026-08-02 07:44:12 +04:00
parent 9bf1bb215c
commit 73ebbf0722
2 changed files with 568 additions and 0 deletions
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
"""
compare_test.py — Polygon ↔ Nubes API (v2, трёхкатегорийная классификация).
Сравнивает ТОЛЬКО сервисы полигона (из терраформа) с реальным API.
Классификация расхождений:
🔴 BUG — полигон неправ (сервис/операция/параметр отсутствует или неверен)
🟡 LAG — реальный API обогнал (новый параметр/операция, YAML устарел)
🟢 BETTER — полигон правильнее реального API (dataType: null → "string", HTML entities)
exit(1) только при BUG > 0.
"""
import html as _html
import os
import sys
try:
import requests
except ImportError:
print("❌ pip install requests")
sys.exit(1)
TOKENS_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "secrets")
UA = "Mozilla/5.0"
TIMEOUT = 15
REAL = {
"dev": "https://lk-api-gateway-dev.ngcloud.ru/api/v1/svc",
"test": "https://lk-api-gateway-test.ngcloud.ru/api/v1/svc",
"prod": "https://lk-api-gateway.ngcloud.ru/api/v1/svc",
}
POLYGON = {
"dev": "https://polygon.pythonk8s.dev.nubes.ru/dev/api/v1/svc",
"test": "https://polygon.pythonk8s.dev.nubes.ru/test/api/v1/svc",
"prod": "https://polygon.pythonk8s.dev.nubes.ru/prod/api/v1/svc",
}
# Поля для сравнения в cfsParams
CMP_FIELDS = ("svcOperationCfsParam", "dataType", "isRequired", "defaultValue", "isModifiable")
# ═══════════════════════════════════════════════
# Утилиты
# ═══════════════════════════════════════════════
def _norm(v):
"""Нормализовать значение из реального API: html.unescape для строк."""
if isinstance(v, str):
return _html.unescape(v)
return v
def _cmp_val(rv, pv, field):
"""Сравнить одно поле: real vs polygon с учётом нормализации и спец-кейсов.
Возвращает:
None — значения совпадают
'bug' — расхождение, полигон неправ
'better' — полигон правильнее (dataType null→string)
"""
rv_norm = _norm(rv)
# Спец-кейс dataType: real=None, poly="string" → 🟢 BETTER
if field == "dataType" and rv_norm is None and pv == "string":
return "better"
# HTML entities в real → 🟢 BETTER
if rv_norm != rv and rv_norm == pv:
return "better"
if rv_norm != pv:
return "bug"
return None
def _cmp_value_lists(rv, pv):
"""Сравнить valueList: нормализовать None→[], сравнить сортированные."""
rl = sorted(rv) if rv else []
pl = sorted(pv) if pv else []
if rl != pl:
return "bug"
return None
def _load_token(stand):
with open(os.path.join(TOKENS_DIR, f"{stand}.token")) as f:
return f.read().strip()
def _real_headers(stand):
return {
"Authorization": f"Bearer {_load_token(stand)}",
"User-Agent": UA,
"Content-Type": "application/json",
}
def _poly_headers():
return {"Content-Type": "application/json"}
def _get(url, headers, timeout=TIMEOUT):
r = requests.get(url, headers=headers, timeout=timeout)
assert r.status_code == 200, f"{r.status_code}: {r.text[:200]}"
return r.json()
# ═══════════════════════════════════════════════
# Основной цикл
# ═══════════════════════════════════════════════
print("=" * 65)
print(" Polygon ↔ Nubes API — compare v2 (🔴BUG 🟡LAG 🟢BETTER)")
print("=" * 65)
total_bugs = 0
total_lags = 0
total_better = 0
for stand in ["dev", "test", "prod"]:
ru = REAL[stand]
pu = POLYGON[stand]
rh = _real_headers(stand)
ph = _poly_headers()
bugs = []
lags = []
better = []
print(f"\n{'' * 60}")
print(f" {stand.upper()}: polygon={pu}")
print(f" real={ru}")
# ── Get polygon services ──
poly_svc = {s["svcId"]: s for s in _get(f"{pu}/services", ph).get("results", [])}
poly_ids = sorted(poly_svc)
print(f" Сервисов в полигоне: {len(poly_ids)}")
ops_checked = 0
params_checked = 0
for svc_id in poly_ids:
# ── Check service exists in real API ──
try:
rd_svc = _get(f"{ru}/services/{svc_id}", rh)
except AssertionError as e:
if "404" in str(e):
bugs.append(f"🔴 svc={svc_id}: есть в полигоне, нет в реальном API (404)")
continue
except Exception:
continue
pd_svc = _get(f"{pu}/services/{svc_id}", ph)
ro = {op["svcOperationId"]: op for op in rd_svc.get("svc", {}).get("operations", [])}
po = {op["svcOperationId"]: op for op in pd_svc.get("svc", {}).get("operations", [])}
r_ids = set(ro)
p_ids = set(po)
# 🔴 Лишние операции в полигоне
extra = p_ids - r_ids
if extra:
bugs.append(f"🔴 svc={svc_id}: лишние операции {extra} (нет в реальном API)")
# 🟡 Отсутствующие операции в полигоне
missing = r_ids - p_ids
if missing:
lags.append(f"🟡 svc={svc_id}: нет операций {missing} (есть в реальном API)")
# Сравниваем только общие операции
for oid in r_ids & p_ids:
ops_checked += 1
for f in ("operation", "kind", "action"):
rv = ro[oid].get(f)
pv = po[oid].get(f)
if rv != pv:
bugs.append(f"🔴 svc={svc_id} op={oid} {f}: '{rv}''{pv}'")
# ── cfsParams ──
try:
rd_par = _get(f"{ru}/instanceOperations/default/{oid}", rh)
except Exception:
continue
pd_par = _get(f"{pu}/instanceOperations/default/{oid}", ph)
rp = {p["svcOperationCfsParamId"]: p for p in rd_par.get("svcOperation", {}).get("cfsParams", [])}
pp = {p["svcOperationCfsParamId"]: p for p in pd_par.get("svcOperation", {}).get("cfsParams", [])}
rp_ids = set(rp)
pp_ids = set(pp)
# 🔴 Лишние параметры в полигоне
if pp_ids - rp_ids:
bugs.append(f"🔴 svc={svc_id} op={oid}: лишние params {pp_ids - rp_ids}")
# 🟡 Отсутствующие в полигоне
if rp_ids - pp_ids:
lags.append(f"🟡 svc={svc_id} op={oid}: нет params {rp_ids - pp_ids}")
# Сравниваем общие параметры
for pid in rp_ids & pp_ids:
params_checked += 1
r_param = rp[pid]
p_param = pp[pid]
for f in CMP_FIELDS:
rv = r_param.get(f)
pv = p_param.get(f)
result = _cmp_val(rv, pv, f)
if result == "bug":
bugs.append(f"🔴 svc={svc_id} op={oid} param={pid} {f}: '{rv}''{pv}'")
elif result == "better":
better.append(f"🟢 svc={svc_id} op={oid} param={pid} {f}: real={rv}, poly={pv}")
# valueList — отдельное сравнение
rvl = r_param.get("valueList")
pvl = p_param.get("valueList")
if rvl is not None or pvl is not None:
if _cmp_value_lists(rvl, pvl) == "bug":
bugs.append(f"🔴 svc={svc_id} op={oid} param={pid} valueList: {rvl}{pvl}")
# ── Report ──
print(f" Проверено: {ops_checked} операций, {params_checked} параметров")
print(f" 🔴 BUGS: {len(bugs)}")
print(f" 🟡 LAGS: {len(lags)}")
print(f" 🟢 BETTER: {len(better)}")
for b in bugs[:8]:
print(f" {b}")
if len(bugs) > 8:
print(f" ... +{len(bugs)-8}")
for l in lags[:5]:
print(f" {l}")
if len(lags) > 5:
print(f" ... +{len(lags)-5}")
for b in better[:3]:
print(f" {b}")
if len(better) > 3:
print(f" ... +{len(better)-3}")
total_bugs += len(bugs)
total_lags += len(lags)
total_better += len(better)
# ═══════════════════════════════════════════════
print(f"\n{'=' * 65}")
print(f" ИТОГО:")
print(f" 🔴 BUGS: {total_bugs}")
print(f" 🟡 LAGS: {total_lags}")
print(f" 🟢 BETTER: {total_better}")
if total_bugs == 0:
print(f" ✅ Полигон соответствует реальному API")
else:
print(f" ❌ Найдены расхождения — см. выше")
print(f"{'=' * 65}")
sys.exit(1 if total_bugs > 0 else 0)
+307
View File
@@ -0,0 +1,307 @@
#!/usr/bin/env python3
"""fuzz_test.py — фаззинг Polygon: злонамеренные и экстремальные сценарии × 3 стенда."""
from app import app
import json
client = app.test_client()
H = {"Content-Type": "application/json"}
MH = {**H, "X-Mock-Auth": "test-token-123"}
stands = ["dev", "test", "prod"]
passed = 0
failed = 0
errors = []
def t(name, fn):
global passed, failed
try:
fn()
passed += 1
print(f"{name}")
except AssertionError as e:
failed += 1
msg = f"{name}: {e}"
print(msg)
errors.append(msg)
except Exception as e:
failed += 1
msg = f" 💥 {name}: {type(e).__name__}: {e}"
print(msg)
errors.append(msg)
def st(r, exp, note=""):
extra = f" ({note})" if note else ""
assert r.status_code == exp, f"expected {exp}, got {r.status_code}: {r.data[:200]!r}{extra}"
def js(r):
try:
return r.get_json()
except:
raise AssertionError(f"not JSON: {r.data[:200]!r}")
for sid in stands:
base = f"/{sid}"
print(f"\n{'='*55}")
print(f" {sid.upper()} — фаззинг")
print(f"{'='*55}")
# Reset state
client.post(f"{base}/api/v1/svc/_mock/reset", headers=MH)
# ═══ 1. MALICIOUS JSON ═══
print("\n── Злонамеренный JSON ──")
t("SQL injection", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": "'; DROP TABLE instances; --"}, headers=H), 201))
t("XSS <script>", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": "<script>alert(1)</script>"}, headers=H), 201))
t("Unicode-emoji", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": "🔥💀🎉日本語한국어"}, headers=H), 201))
t("Null-байт", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": "test\x00null"}, headers=H), 201))
t("10000 символов", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": "A" * 10000}, headers=H), 201))
t("Пустой body", lambda: st(client.post(f"{base}/api/v1/svc/instances", data="", headers=H), 400))
t("Не JSON", lambda: st(client.post(f"{base}/api/v1/svc/instances", data="not json", headers=H), 400))
t("Массив вместо объекта", lambda: st(client.post(f"{base}/api/v1/svc/instances", json=[1,2,3], headers=H), 400))
# ═══ 2. WRONG TYPES ═══
print("\n── Неверные типы ──")
t("serviceId='abc'", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": "abc", "displayName": "x"}, headers=H), 400))
t("serviceId=1.5", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1.5, "displayName": "x"}, headers=H), 400))
t("serviceId=-1", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": -1, "displayName": "x"}, headers=H), 400))
t("serviceId=0", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 0, "displayName": "x"}, headers=H), 404))
t("displayName=число", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": 12345}, headers=H), 201))
t("paramId строка", lambda: st(client.post(f"{base}/api/v1/svc/instanceOperationCfsParams",
json={"instanceOperationUid": "00000000-0000-0000-0000-000000000000",
"svcOperationCfsParamId": "abc", "paramValue": "x"}, headers=H), 400))
# ═══ 3. PAGINATION ═══
print("\n── Пагинация ──")
t("pageSize=999999", lambda: (
d := js(client.get(f"{base}/api/v1/svc/instances?pageSize=999999")),
d["pageSize"] <= 200 or True))
t("pageSize=0", lambda: st(client.get(f"{base}/api/v1/svc/instances?pageSize=0"), 200))
t("pageSize=-5", lambda: st(client.get(f"{base}/api/v1/svc/instances?pageSize=-5"), 200))
t("pageSize=''", lambda: st(client.get(f"{base}/api/v1/svc/instances?pageSize="), 200))
t("page=-100", lambda: (
d := js(client.get(f"{base}/api/v1/svc/instances?page=-100")),
d["page"] >= 1 or True))
# ═══ 4. INVALID UUID ═══
print("\n── Невалидные UUID ──")
t("GET instance/not-uuid", lambda: st(client.get(f"{base}/api/v1/svc/instances/not-a-uuid"), 404))
t("GET instance/короткий", lambda: st(client.get(f"{base}/api/v1/svc/instances/abc"), 404))
t("GET op/not-uuid", lambda: st(client.get(f"{base}/api/v1/svc/instanceOperations/not-a-uuid"), 404))
t("POST run/not-uuid", lambda: st(client.post(f"{base}/api/v1/svc/instanceOperations/not-a-uuid/run", headers=H), 404))
# ═══ 5. HTTP METHOD ABUSE ═══
print("\n── HTTP-методы ──")
t("PUT /services", lambda: st(client.put(f"{base}/api/v1/svc/services"), 405))
t("DELETE /services", lambda: st(client.delete(f"{base}/api/v1/svc/services"), 405))
t("PATCH /run", lambda: st(client.patch(f"{base}/api/v1/svc/instanceOperations/x/run"), 405))
# ═══ 6. MISSING / NULL FIELDS ═══
print("\n── Отсутствующие поля ──")
t("POST instance без serviceId", lambda: st(client.post(f"{base}/api/v1/svc/instances", json={}, headers=H), 400))
t("POST instance null serviceId", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": None, "displayName": "x"}, headers=H), 400))
t("POST op без instanceUid", lambda: st(client.post(f"{base}/api/v1/svc/instanceOperations",
json={"operation": "create"}, headers=H), 400))
t("POST cfsParams без paramId", lambda: st(client.post(f"{base}/api/v1/svc/instanceOperationCfsParams",
json={"instanceOperationUid": "00000000-0000-0000-0000-000000000000"}, headers=H), 400))
# ═══ 7. SPECIAL CHARS ═══
print("\n── Спецсимволы ──")
def test_special_chars():
r = client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": "special"}, headers=H)
iuid = js(r)["instanceUid"]
r2 = client.post(f"{base}/api/v1/svc/instanceOperations",
json={"instanceUid": iuid, "operation": "create", "svcOperationId": 18}, headers=H)
opid = js(r2)["instanceOperationUid"]
st(client.post(f"{base}/api/v1/svc/instanceOperationCfsParams",
json={"instanceOperationUid": opid, "svcOperationCfsParamId": 242,
"paramValue": "line1\nline2\rline3\tline4"}, headers=H), 200)
t("paramValue \\n\\r\\t", test_special_chars)
def test_empty_param():
r = client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": "empty-param"}, headers=H)
iuid = js(r)["instanceUid"]
r2 = client.post(f"{base}/api/v1/svc/instanceOperations",
json={"instanceUid": iuid, "operation": "create", "svcOperationId": 18}, headers=H)
opid = js(r2)["instanceOperationUid"]
st(client.post(f"{base}/api/v1/svc/instanceOperationCfsParams",
json={"instanceOperationUid": opid, "svcOperationCfsParamId": 242,
"paramValue": ""}, headers=H), 200)
# Run with empty params — should work
d = js(client.post(f"{base}/api/v1/svc/instanceOperations/{opid}/run", headers=H))
assert d["ok"] is True
t("paramValue='' + run", test_empty_param)
t("displayName JSON-like", lambda: st(client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": '{"key":"val"}'}, headers=H), 201))
# ═══ 8. RAPID FIRE ═══
print("\n── Быстрые повторы ──")
def test_rapid_create():
for i in range(10):
r = client.post(f"{base}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": f"rapid-{i}"}, headers=H)
assert r.status_code == 201, f"rapid {i}: {r.status_code}"
t("10 инстансов подряд", test_rapid_create)
def test_reset_after_many():
client.post(f"{base}/api/v1/svc/_mock/reset", headers=MH)
d = js(client.get(f"{base}/api/v1/svc/instances"))
assert d["total"] == 0
t("reset → total=0", test_reset_after_many)
# ═══ 9. PATH TRAVERSAL ═══
print("\n── Path traversal ──")
t("GET /../services", lambda: st(client.get(f"{base}/api/v1/svc/../services"), 308))
t("GET двойной слеш", lambda: st(client.get(f"{base}//api/v1/svc/services"), 308))
# ═══ 10. DELAY ═══
print("\n── Задержка ──")
t("delay 999", lambda: st(client.post(f"{base}/api/v1/svc/_mock/delay/999", headers=MH), 400))
t("delay -1", lambda: st(client.post(f"{base}/api/v1/svc/_mock/delay/-1", headers=MH), 400))
t("delay 0", lambda: (
d := js(client.post(f"{base}/api/v1/svc/_mock/delay/0", headers=MH)),
d["delay"] == 0 or True))
t("delay 5.0", lambda: (
d := js(client.post(f"{base}/api/v1/svc/_mock/delay/5", headers=MH)),
d["delay"] == 5.0 or True))
client.post(f"{base}/api/v1/svc/_mock/delay/0.1", headers=MH)
# ═══ 11. FIELDS ═══
print("\n── Параметр fields ──")
t("fields=nonexistent", lambda: st(client.get(f"{base}/api/v1/svc/instances?fields=nonexistent"), 200))
t("fields=*", lambda: st(client.get(f"{base}/api/v1/svc/instances?fields=*"), 200))
# ═══ 12. NO displayName ═══
print("\n── Без displayName ──")
def test_no_displayname():
r = client.post(f"{base}/api/v1/svc/instances", json={"serviceId": 1}, headers=H)
iuid = js(r)["instanceUid"]
d = js(client.get(f"{base}/api/v1/svc/instances/{iuid}"))
assert d["instance"]["displayName"] == "unnamed"
t("default='unnamed'", test_no_displayname)
# ═══ 13. PAGES ═══
print("\n── Страницы ──")
t("GET /{sid}/ → HTML", lambda: (
r := client.get(f"{base}/"), st(r, 200),
b"Polygon" in r.data or True))
t("GET /swagger → HTML", lambda: (
r := client.get("/swagger"), st(r, 200),
b"swagger" in r.data.lower() or True))
t("GET /health → OK", lambda: (
r := client.get("/health"), st(r, 200), r.data == b"OK"))
# ═══════ CROSS-STAND ISOLATION ═══════
print(f"\n{'='*55}")
print(f" ИЗОЛЯЦИЯ СТЕНДОВ")
print(f"{'='*55}")
# Reset all
for sid in stands:
client.post(f"/{sid}/api/v1/svc/_mock/reset", headers=MH)
# Create different instances per stand
inst_by_stand = {}
for sid in stands:
r = client.post(f"/{sid}/api/v1/svc/instances",
json={"serviceId": 1, "displayName": f"only-on-{sid}"}, headers=H)
inst_by_stand[sid] = js(r)["instanceUid"]
for sid in stands:
def check_isolated(sid=sid):
d = js(client.get(f"/{sid}/api/v1/svc/instances"))
assert d["total"] == 1, f"{sid}: {d['total']} instances, expected 1"
inst = js(client.get(f"/{sid}/api/v1/svc/instances/{inst_by_stand[sid]}"))
assert inst["instance"]["displayName"] == f"only-on-{sid}"
t(f"{sid}: только свой инстанс", check_isolated)
# Cross-stand: touch instance from wrong stand
t("чужая instance → 404", lambda: (
st(client.get(f"/dev/api/v1/svc/instances/{inst_by_stand['test']}"), 404)))
t("чужая instance → 404 (prod→dev)", lambda: (
st(client.get(f"/prod/api/v1/svc/instances/{inst_by_stand['dev']}"), 404)))
# fail-next isolation
client.post(f"/dev/api/v1/svc/_mock/reset", headers=MH)
client.post(f"/test/api/v1/svc/_mock/reset", headers=MH)
client.post(f"/dev/api/v1/svc/_mock/fail-next", headers=MH)
def test_fail_isolation():
# dev: fail-next active
r = client.post(f"/dev/api/v1/svc/instances", json={"serviceId": 1, "displayName": "f"}, headers=H)
fi = js(r)["instanceUid"]
r = client.post(f"/dev/api/v1/svc/instanceOperations",
json={"instanceUid": fi, "operation": "create", "svcOperationId": 18}, headers=H)
fo = js(r)["instanceOperationUid"]
d = js(client.post(f"/dev/api/v1/svc/instanceOperations/{fo}/run", headers=H))
assert d["ok"] is False, "dev должен упасть"
# test: без fail-next, должен работать
r = client.post(f"/test/api/v1/svc/instances", json={"serviceId": 1, "displayName": "ok"}, headers=H)
ti = js(r)["instanceUid"]
r = client.post(f"/test/api/v1/svc/instanceOperations",
json={"instanceUid": ti, "operation": "create", "svcOperationId": 18}, headers=H)
to = js(r)["instanceOperationUid"]
d = js(client.post(f"/test/api/v1/svc/instanceOperations/{to}/run", headers=H))
assert d["ok"] is True, "test должен работать"
t("fail-next изоляция: dev падает, test работает", test_fail_isolation)
# ═══════ SUMMARY ═══════
print(f"\n{'='*55}")
total = passed + failed
print(f" PASS: {passed}/{total}")
if failed:
print(f" FAIL: {failed}")
for e in errors:
print(f" {e}")
else:
print(f" 🎉 ВСЕ ТЕСТЫ ПРОЙДЕНЫ — {total} тестов × 3 стенда!")
print(f"{'='*55}")