HISTORY: результаты диагностики платформы (зависание больших POST ~51с, WAF) + platform_probe
This commit is contained in:
“Naeel”
@@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env python3
|
||||
"""tests/platform_probe.py — диагностика транспортного слоя платформы (HTTP контейнер).
|
||||
|
||||
Проверяет, как платформа (шлюз/ingress/DDoS-Guard) пропускает трафик к сервису,
|
||||
НЕЗАВИСИМО от бизнес-логики SQS. Запускать и на ВМ, и на локальной машине
|
||||
для сравнения путей. Только stdlib + boto3.
|
||||
|
||||
Блоки:
|
||||
A — через /health: burst 500 параллельных, большие заголовки, медленное тело,
|
||||
keep-alive x50, HTTP/1.0, HEAD, POST.
|
||||
B — через SQS API (транспорт): SendMessage 256KB, Receive 10x256KB (~2.5MB ответ),
|
||||
long-poll 20s (фактическая длительность), chunked-тело.
|
||||
"""
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import ssl
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
ENDPOINT = os.environ.get("ENDPOINT_URL", "https://sqs.containerk8s.dev.nubes.ru")
|
||||
HOST = ENDPOINT.replace("https://", "").replace("http://", "").rstrip("/")
|
||||
PORT = 443
|
||||
REGION = os.environ.get("REGION", "us-east-1")
|
||||
MODE = os.environ.get("PROBE_MODE", "unknown")
|
||||
|
||||
results = {}
|
||||
|
||||
|
||||
def _conn():
|
||||
c = http.client.HTTPSConnection(HOST, PORT, timeout=30)
|
||||
return c
|
||||
|
||||
|
||||
def record(name, value):
|
||||
results[name] = value
|
||||
print("%s: %s" % (name, json.dumps(value, ensure_ascii=False)), flush=True)
|
||||
|
||||
|
||||
def http_get(path, headers=None, method="GET", body=None):
|
||||
c = _conn()
|
||||
try:
|
||||
c.request(method, path, body=body, headers=headers or {})
|
||||
r = c.getresponse()
|
||||
data = r.read()
|
||||
return r.status, dict(r.getheaders()), data
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
|
||||
# --- A1: burst 500 параллельных /health ---
|
||||
def a1_burst():
|
||||
n = 500
|
||||
ok = err = 0
|
||||
lat = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def one():
|
||||
nonlocal ok, err
|
||||
t0 = time.time()
|
||||
try:
|
||||
st, _, _ = http_get("/health")
|
||||
with lock:
|
||||
lat.append(time.time() - t0)
|
||||
if st == 200:
|
||||
ok += 1
|
||||
else:
|
||||
err += 1
|
||||
except Exception:
|
||||
with lock:
|
||||
err += 1
|
||||
|
||||
threads = [threading.Thread(target=one) for _ in range(n)]
|
||||
t0 = time.time()
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
total = time.time() - t0
|
||||
lat.sort()
|
||||
record("A1_burst_500", {
|
||||
"ok": ok, "err": err, "total_s": round(total, 2),
|
||||
"p50": round(lat[len(lat) // 2], 3) if lat else None,
|
||||
"p95": round(lat[int(len(lat) * 0.95)], 3) if lat else None,
|
||||
"max": round(lat[-1], 3) if lat else None,
|
||||
})
|
||||
|
||||
|
||||
# --- A2: большие заголовки ---
|
||||
def a2_big_headers():
|
||||
for kb in (8, 16, 64):
|
||||
h = {"User-Agent": "platform-probe", "X-Pad": "x" * (kb * 1024)}
|
||||
try:
|
||||
st, _, _ = http_get("/health", headers=h)
|
||||
record("A2_header_%dkb" % kb, st)
|
||||
except Exception as e:
|
||||
record("A2_header_%dkb" % kb, type(e).__name__ + ": " + str(e)[:80])
|
||||
|
||||
|
||||
# --- A3: медленное тело (1 байт / 0.5с, Content-Length 10) ---
|
||||
def a3_slow_body():
|
||||
try:
|
||||
ctx = ssl.create_default_context()
|
||||
raw = ctx.wrap_socket(socket.create_connection((HOST, PORT), timeout=15),
|
||||
server_hostname=HOST)
|
||||
raw.sendall(b"POST /health HTTP/1.1\r\nHost: %s\r\nContent-Length: 10\r\nConnection: close\r\n\r\n" % HOST.encode())
|
||||
t0 = time.time()
|
||||
for _ in range(10):
|
||||
raw.sendall(b"x")
|
||||
time.sleep(0.5)
|
||||
resp = raw.recv(65536)
|
||||
dt = time.time() - t0
|
||||
raw.close()
|
||||
first_line = resp.split(b"\r\n", 1)[0].decode("latin1", "replace") if resp else "(no response)"
|
||||
record("A3_slow_body", {"resp": first_line, "t_s": round(dt, 2)})
|
||||
except Exception as e:
|
||||
record("A3_slow_body", type(e).__name__ + ": " + str(e)[:80])
|
||||
|
||||
|
||||
# --- A4: keep-alive 50 запросов на одном соединении ---
|
||||
def a4_keepalive():
|
||||
ok = err = 0
|
||||
c = _conn()
|
||||
try:
|
||||
for _ in range(50):
|
||||
try:
|
||||
c.request("GET", "/health")
|
||||
r = c.getresponse()
|
||||
r.read()
|
||||
if r.status == 200:
|
||||
ok += 1
|
||||
else:
|
||||
err += 1
|
||||
except Exception:
|
||||
err += 1
|
||||
break
|
||||
finally:
|
||||
c.close()
|
||||
record("A4_keepalive_50", {"ok": ok, "err": err})
|
||||
|
||||
|
||||
# --- A5: HTTP/1.0, HEAD, POST ---
|
||||
def a5_variants():
|
||||
try:
|
||||
ctx = ssl.create_default_context()
|
||||
raw = ctx.wrap_socket(socket.create_connection((HOST, PORT), timeout=15),
|
||||
server_hostname=HOST)
|
||||
raw.sendall(b"GET /health HTTP/1.0\r\nHost: %s\r\n\r\n" % HOST.encode())
|
||||
resp = raw.recv(4096)
|
||||
raw.close()
|
||||
record("A5_http1.0", resp.split(b"\r\n", 1)[0].decode("latin1", "replace") if resp else "(no response)")
|
||||
except Exception as e:
|
||||
record("A5_http1.0", type(e).__name__ + ": " + str(e)[:80])
|
||||
try:
|
||||
st, _, _ = http_get("/health", method="HEAD")
|
||||
record("A5_head", st)
|
||||
except Exception as e:
|
||||
record("A5_head", type(e).__name__ + ": " + str(e)[:80])
|
||||
try:
|
||||
st, _, _ = http_get("/health", method="POST", body=b"x")
|
||||
record("A5_post", st)
|
||||
except Exception as e:
|
||||
record("A5_post", type(e).__name__ + ": " + str(e)[:80])
|
||||
|
||||
|
||||
# --- B6: SendMessage 256KB ---
|
||||
def b6_big_send(sqs, url):
|
||||
t0 = time.time()
|
||||
try:
|
||||
sqs.send_message(QueueUrl=url, MessageBody="x" * (256 * 1024))
|
||||
record("B6_send_256kb", {"ok": True, "t_s": round(time.time() - t0, 3)})
|
||||
except Exception as e:
|
||||
record("B6_send_256kb", {"ok": False, "err": type(e).__name__ + ": " + str(e)[:80], "t_s": round(time.time() - t0, 3)})
|
||||
|
||||
|
||||
# --- B8: Receive 10 x 256KB (~2.5MB ответ) ---
|
||||
def b8_big_receive(sqs, url):
|
||||
for _ in range(10):
|
||||
sqs.send_message(QueueUrl=url, MessageBody="y" * (256 * 1024))
|
||||
t0 = time.time()
|
||||
try:
|
||||
r = sqs.receive_message(QueueUrl=url, MaxNumberOfMessages=10, WaitTimeSeconds=1)
|
||||
msgs = r.get("Messages", [])
|
||||
total = sum(len(m["Body"]) for m in msgs)
|
||||
record("B8_receive_2.5mb", {"ok": True, "msgs": len(msgs), "bytes": total, "t_s": round(time.time() - t0, 3)})
|
||||
for m in msgs:
|
||||
try:
|
||||
sqs.delete_message(QueueUrl=url, ReceiptHandle=m["ReceiptHandle"])
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
record("B8_receive_2.5mb", {"ok": False, "err": type(e).__name__ + ": " + str(e)[:80], "t_s": round(time.time() - t0, 3)})
|
||||
|
||||
|
||||
# --- B9: long-poll 20s — фактическая длительность ---
|
||||
def b9_longpoll(sqs, url):
|
||||
t0 = time.time()
|
||||
try:
|
||||
sqs.receive_message(QueueUrl=url, MaxNumberOfMessages=1, WaitTimeSeconds=20)
|
||||
record("B9_longpoll_20s", {"ok": True, "t_s": round(time.time() - t0, 2)})
|
||||
except Exception as e:
|
||||
record("B9_longpoll_20s", {"ok": False, "err": type(e).__name__ + ": " + str(e)[:80], "t_s": round(time.time() - t0, 2)})
|
||||
|
||||
|
||||
# --- B7: chunked-тело ---
|
||||
def b7_chunked():
|
||||
try:
|
||||
ctx = ssl.create_default_context()
|
||||
raw = ctx.wrap_socket(socket.create_connection((HOST, PORT), timeout=15),
|
||||
server_hostname=HOST)
|
||||
raw.sendall(b"POST /health HTTP/1.1\r\nHost: %s\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" % HOST.encode())
|
||||
raw.sendall(b"5\r\nhello\r\n")
|
||||
raw.sendall(b"5\r\nworld\r\n")
|
||||
raw.sendall(b"0\r\n\r\n")
|
||||
resp = raw.recv(4096)
|
||||
raw.close()
|
||||
record("B7_chunked", resp.split(b"\r\n", 1)[0].decode("latin1", "replace") if resp else "(no response)")
|
||||
except Exception as e:
|
||||
record("B7_chunked", type(e).__name__ + ": " + str(e)[:80])
|
||||
|
||||
|
||||
def main():
|
||||
sqs = boto3.client("sqs", endpoint_url=ENDPOINT, region_name=REGION,
|
||||
config=Config(connect_timeout=10, read_timeout=30, retries={"max_attempts": 1}))
|
||||
q = "platprobe-%s" % int(time.time())
|
||||
url = sqs.create_queue(QueueName=q)["QueueUrl"]
|
||||
print("queue: %s" % url, flush=True)
|
||||
try:
|
||||
a1_burst()
|
||||
a2_big_headers()
|
||||
a3_slow_body()
|
||||
a4_keepalive()
|
||||
a5_variants()
|
||||
b6_big_send(sqs, url)
|
||||
b7_chunked()
|
||||
b8_big_receive(sqs, url)
|
||||
b9_longpoll(sqs, url)
|
||||
finally:
|
||||
try:
|
||||
sqs.delete_queue(QueueUrl=url)
|
||||
except Exception:
|
||||
pass
|
||||
print("MODE=%s ENDPOINT=%s" % (MODE, ENDPOINT), flush=True)
|
||||
print("SUMMARY: %s" % json.dumps(results, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user