test(loadtests): 10 load scenarios + verifier; docs: Sonnet review findings
This commit is contained in:
“Naeel”
@@ -0,0 +1,430 @@
|
||||
# scenarios.py — нагрузочные сценарии.
|
||||
#
|
||||
# Каждый сценарий: prepare → run → verify → report (dict) → cleanup.
|
||||
# Все устройства создаются самим тестом и помечены run_id; cleanup удаляет
|
||||
# только их (если IOT_CLEANUP=1).
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
|
||||
from .common import API, CLEANUP, Counter, NS_PREFIX, latency_stats, now_ms
|
||||
from .publisher import DevicePublisher
|
||||
from .verifier import Verifier
|
||||
|
||||
|
||||
def _ns(run_id):
|
||||
return f"{NS_PREFIX}_{run_id}"
|
||||
|
||||
|
||||
def _create_devices(api, ns, run_id, count, device_prefix="dev"):
|
||||
devices = []
|
||||
for i in range(count):
|
||||
name = f"{device_prefix}{i}"
|
||||
d = api.create_device(ns, name, f"{device_prefix}-{i}")
|
||||
# POST возвращает устройство БЕЗ пароля — пароль только в GET по имени.
|
||||
d = api.get_device(ns, name)
|
||||
devices.append({
|
||||
"name": name,
|
||||
"device_id": d["device_id"],
|
||||
"username": d["mqtt_username"],
|
||||
"password": d["mqtt_password"],
|
||||
})
|
||||
return devices
|
||||
|
||||
|
||||
def _cleanup(api, ns, devices):
|
||||
if not CLEANUP:
|
||||
return
|
||||
for d in devices:
|
||||
try:
|
||||
api.delete_device(ns, d["name"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _run_publishers(run_id, ns, devices, **kw):
|
||||
"""Запускает паблишеры, ждёт завершения, возвращает счётчики."""
|
||||
counter = Counter()
|
||||
pubs = [DevicePublisher(run_id, ns, d["device_id"], d["username"],
|
||||
d["password"], counter=counter, **kw)
|
||||
for d in devices]
|
||||
for p in pubs:
|
||||
p.start()
|
||||
for p in pubs:
|
||||
p.join()
|
||||
return counter.snapshot()
|
||||
|
||||
|
||||
def _wait_delivery(api, ns, run_id, devices, expected_total, timeout=120):
|
||||
"""Ждёт, пока доедет expected_total сообщений (или таймаут)."""
|
||||
v = Verifier(api, ns, run_id)
|
||||
t0 = time.monotonic()
|
||||
while time.monotonic() - t0 < timeout:
|
||||
per = sum(v.device_report(d["device_id"], 0)["delivered"]
|
||||
for d in devices)
|
||||
rep = v.run_report(devices, expected_per_device=0)
|
||||
if rep["delivered"] >= expected_total:
|
||||
return rep, time.monotonic() - t0
|
||||
time.sleep(5)
|
||||
return v.run_report(devices, expected_per_device=0), time.monotonic() - t0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 1. BASELINE — равномерная нагрузка
|
||||
# --------------------------------------------------------------------------
|
||||
def baseline(args):
|
||||
run_id = f"base-{int(time.time())}"
|
||||
ns = _ns(run_id)
|
||||
api = API()
|
||||
devices = _create_devices(api, ns, run_id, args.devices)
|
||||
try:
|
||||
expected_per_device = max(1, int(args.rate * args.duration))
|
||||
t0 = now_ms()
|
||||
counters = _run_publishers(
|
||||
run_id, ns, devices, rate=args.rate, duration=args.duration,
|
||||
qos=args.qos, payload_size=args.payload_size,
|
||||
reconnect_every=args.reconnect_every)
|
||||
send_ms = now_ms() - t0
|
||||
time.sleep(10) # consumer long-poll до ~20с
|
||||
rep = Verifier(api, ns, run_id).run_report(
|
||||
devices, expected_per_device)
|
||||
rep.update({
|
||||
"scenario": "baseline",
|
||||
"ns": ns,
|
||||
"send_ms": send_ms,
|
||||
"publisher": counters,
|
||||
"rate_rps": round(counters.get("sent", 0) / max(1, send_ms / 1000), 2),
|
||||
})
|
||||
return rep
|
||||
finally:
|
||||
_cleanup(api, ns, devices)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 2. BURST — всплеск
|
||||
# --------------------------------------------------------------------------
|
||||
def burst(args):
|
||||
run_id = f"burst-{int(time.time())}"
|
||||
ns = _ns(run_id)
|
||||
api = API()
|
||||
devices = _create_devices(api, ns, run_id, args.devices)
|
||||
try:
|
||||
quiet = 20
|
||||
spike = args.duration
|
||||
low = _run_publishers(run_id + "-l", ns, devices,
|
||||
rate=args.rate, duration=quiet, qos=args.qos)
|
||||
high = _run_publishers(run_id + "-h", ns, devices,
|
||||
rate=args.rate * args.burst_mult,
|
||||
duration=spike, qos=args.qos)
|
||||
tail = _run_publishers(run_id + "-t", ns, devices,
|
||||
rate=args.rate, duration=quiet, qos=args.qos)
|
||||
expected_low = max(1, int(args.rate * quiet))
|
||||
expected_high = max(1, int(args.rate * args.burst_mult * spike))
|
||||
rep = Verifier(api, ns, run_id)
|
||||
# считаем только фазу всплеска (низкие фазы — фон)
|
||||
rep_high = rep.run_report(devices, expected_high)
|
||||
# recovery: сколько времени после окончания всплеска очередь отдала всё
|
||||
_, recovery_s = _wait_delivery(
|
||||
api, ns, run_id + "-h", devices, expected_high * len(devices),
|
||||
timeout=300)
|
||||
return {
|
||||
"scenario": "burst",
|
||||
"ns": ns,
|
||||
"quiet_phase": low,
|
||||
"spike_phase": high,
|
||||
"tail_phase": tail,
|
||||
"expected_high_total": expected_high * len(devices),
|
||||
"recovery_sec": round(recovery_s, 1),
|
||||
"high": rep_high,
|
||||
}
|
||||
finally:
|
||||
_cleanup(api, ns, devices)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 3. LARGE_PAYLOAD — крупные сообщения
|
||||
# --------------------------------------------------------------------------
|
||||
def large_payload(args):
|
||||
run_id = f"big-{int(time.time())}"
|
||||
ns = _ns(run_id)
|
||||
api = API()
|
||||
devices = _create_devices(api, ns, run_id, min(args.devices, 10))
|
||||
try:
|
||||
expected = max(1, int(args.rate * args.duration))
|
||||
counters = _run_publishers(
|
||||
run_id, ns, devices, rate=args.rate, duration=args.duration,
|
||||
qos=args.qos, payload_size=args.payload_size)
|
||||
time.sleep(10)
|
||||
rep = Verifier(api, ns, run_id).run_report(devices, expected)
|
||||
rep.update({"scenario": "large_payload", "ns": ns,
|
||||
"publisher": counters})
|
||||
return rep
|
||||
finally:
|
||||
_cleanup(api, ns, devices)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 4. RECONNECT_STORM — массовые переподключения
|
||||
# --------------------------------------------------------------------------
|
||||
def reconnect_storm(args):
|
||||
run_id = f"rc-{int(time.time())}"
|
||||
ns = _ns(run_id)
|
||||
api = API()
|
||||
devices = _create_devices(api, ns, run_id, args.devices)
|
||||
try:
|
||||
expected = max(1, int(args.rate * args.duration))
|
||||
counters = _run_publishers(
|
||||
run_id, ns, devices, rate=args.rate, duration=args.duration,
|
||||
qos=1, reconnect_every=args.reconnect_every)
|
||||
time.sleep(10)
|
||||
rep = Verifier(api, ns, run_id).run_report(devices, expected)
|
||||
rep.update({
|
||||
"scenario": "reconnect_storm",
|
||||
"ns": ns,
|
||||
"publisher": counters,
|
||||
"reconnect_every_sec": args.reconnect_every,
|
||||
})
|
||||
return rep
|
||||
finally:
|
||||
_cleanup(api, ns, devices)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 5. MULTITENANT — много namespace, задержка первого сообщения
|
||||
# --------------------------------------------------------------------------
|
||||
def multitenant(args):
|
||||
run_id = f"mt-{int(time.time())}"
|
||||
api = API()
|
||||
devices_per_tenant = args.devices_per_tenant
|
||||
all_devices = []
|
||||
tenant_devices = {}
|
||||
for t in range(args.tenants):
|
||||
ns = _ns(f"{run_id}-{t}")
|
||||
devs = _create_devices(api, ns, run_id, devices_per_tenant,
|
||||
device_prefix=f"t{t}")
|
||||
tenant_devices[ns] = devs
|
||||
all_devices.extend(devs)
|
||||
try:
|
||||
expected = max(1, int(args.rate * args.duration))
|
||||
t0 = now_ms()
|
||||
for ns, devs in tenant_devices.items():
|
||||
_run_publishers(run_id, ns, devs, rate=args.rate,
|
||||
duration=args.duration, qos=args.qos)
|
||||
time.sleep(15)
|
||||
first_lats = []
|
||||
for ns, devs in tenant_devices.items():
|
||||
v = Verifier(api, ns, run_id)
|
||||
for d in devs:
|
||||
r = v.device_report(d["device_id"], expected)
|
||||
if r["first_latency_ms"] is not None:
|
||||
first_lats.append(r["first_latency_ms"])
|
||||
return {
|
||||
"scenario": "multitenant",
|
||||
"tenants": args.tenants,
|
||||
"devices_per_tenant": devices_per_tenant,
|
||||
"total_devices": len(all_devices),
|
||||
"elapsed_ms": now_ms() - t0,
|
||||
"first_msg_latency_ms": latency_stats(first_lats),
|
||||
}
|
||||
finally:
|
||||
for ns, devs in tenant_devices.items():
|
||||
_cleanup(api, ns, devs)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 6. ACL_VIOLATION — публикация в чужой топик
|
||||
# --------------------------------------------------------------------------
|
||||
def acl_violation(args):
|
||||
run_id = f"acl-{int(time.time())}"
|
||||
ns = _ns(run_id)
|
||||
api = API()
|
||||
devices = _create_devices(api, ns, run_id, 2)
|
||||
try:
|
||||
victim_ns = _ns(f"{run_id}-victim")
|
||||
victim = _create_devices(api, victim_ns, run_id, 1)[0]
|
||||
attacker = devices[0]
|
||||
foreign_topic = f"{victim_ns}/telemetry/{victim['device_id']}"
|
||||
counter = Counter()
|
||||
p = DevicePublisher(run_id, ns, attacker["device_id"],
|
||||
attacker["username"], attacker["password"],
|
||||
rate=1, duration=10, qos=0, counter=counter,
|
||||
extra_topic=foreign_topic, publish_foreign=True)
|
||||
p.start()
|
||||
p.join()
|
||||
return {
|
||||
"scenario": "acl_violation",
|
||||
"ns": ns,
|
||||
"publisher": counter.snapshot(),
|
||||
"expect": "foreign_denied=все, foreign_accepted=0 "
|
||||
"(EMQX рвёт сессию: deny_action=disconnect)",
|
||||
}
|
||||
finally:
|
||||
_cleanup(api, ns, devices)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 7. AUTH_NEG — отказ на неверные креды
|
||||
# --------------------------------------------------------------------------
|
||||
def auth_neg(args):
|
||||
import paho.mqtt.client as mqtt
|
||||
|
||||
from .common import WS_HOST, WS_PATH, WS_PORT, ws_tls_opts
|
||||
|
||||
def attempt(client_id, username, password):
|
||||
res = {}
|
||||
c = mqtt.Client(client_id=client_id, transport="websockets")
|
||||
c.ws_set_options(path=WS_PATH)
|
||||
c.username_pw_set(username, password)
|
||||
ws_tls_opts(c)
|
||||
|
||||
def on_connect(cl, ud, flags, rc, props=None):
|
||||
res["rc"] = rc
|
||||
c.on_connect = on_connect
|
||||
try:
|
||||
c.connect(WS_HOST, WS_PORT, 15)
|
||||
c.loop_start()
|
||||
time.sleep(2)
|
||||
c.loop_stop()
|
||||
except Exception as e:
|
||||
res["exc"] = type(e).__name__
|
||||
return res
|
||||
|
||||
results = {
|
||||
"wrong_password": attempt("lt-neg-pass", "test_dev-001", "wrong-123"),
|
||||
"unknown_user": attempt("lt-neg-user", "nosuch_user", "whatever"),
|
||||
}
|
||||
return {"scenario": "auth_neg",
|
||||
"expect": "wrong_password rc=4, unknown_user rc=5",
|
||||
"results": results}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 8. API_CRUD — нагрузка на CRUD устройств
|
||||
# --------------------------------------------------------------------------
|
||||
def api_crud(args):
|
||||
run_id = f"api-{int(time.time())}"
|
||||
ns = _ns(run_id)
|
||||
api = API()
|
||||
lat = {"POST": [], "GET": [], "DELETE": []}
|
||||
lock = threading.Lock()
|
||||
errors = Counter()
|
||||
|
||||
def worker(wid):
|
||||
for i in range(args.iterations):
|
||||
name = f"w{wid}-{i}"
|
||||
for op, fn in (
|
||||
("POST", lambda: api.create_device(ns, name, f"w{wid}-{i}")),
|
||||
("GET", lambda: api.get_device(ns, name)),
|
||||
("DELETE", lambda: api.delete_device(ns, name)),
|
||||
):
|
||||
t0 = now_ms()
|
||||
try:
|
||||
fn()
|
||||
with lock:
|
||||
lat[op].append(now_ms() - t0)
|
||||
except Exception:
|
||||
errors.inc(op)
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(w,))
|
||||
for w in range(args.threads)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
return {
|
||||
"scenario": "api_crud",
|
||||
"ns": ns,
|
||||
"threads": args.threads,
|
||||
"iterations_per_thread": args.iterations,
|
||||
"POST": latency_stats(lat["POST"]),
|
||||
"GET": latency_stats(lat["GET"]),
|
||||
"DELETE": latency_stats(lat["DELETE"]),
|
||||
"errors": errors.snapshot(),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 9. TELEMETRY_QUERY — нагрузка на чтение телеметрии
|
||||
# --------------------------------------------------------------------------
|
||||
def telemetry_query(args):
|
||||
run_id = f"tq-{int(time.time())}"
|
||||
ns = _ns(run_id)
|
||||
api = API()
|
||||
devices = _create_devices(api, ns, run_id, max(1, args.devices))
|
||||
try:
|
||||
# сидируем данные
|
||||
_run_publishers(run_id, ns, devices, rate=5, duration=20, qos=0)
|
||||
time.sleep(10)
|
||||
lat = []
|
||||
lock = threading.Lock()
|
||||
errors = Counter()
|
||||
|
||||
def worker(wid):
|
||||
for _ in range(args.iterations):
|
||||
t0 = now_ms()
|
||||
try:
|
||||
api.telemetry(ns, limit=args.query_limit)
|
||||
with lock:
|
||||
lat.append(now_ms() - t0)
|
||||
except Exception:
|
||||
errors.inc("query")
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(w,))
|
||||
for w in range(args.threads)]
|
||||
t0 = now_ms()
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
elapsed = max(1, (now_ms() - t0) / 1000)
|
||||
return {
|
||||
"scenario": "telemetry_query",
|
||||
"ns": ns,
|
||||
"threads": args.threads,
|
||||
"iterations": args.iterations,
|
||||
"query_limit": args.query_limit,
|
||||
"req_per_sec": round(len(lat) / elapsed, 2),
|
||||
"latency": latency_stats(lat),
|
||||
"errors": errors.snapshot(),
|
||||
}
|
||||
finally:
|
||||
_cleanup(api, ns, devices)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 10. SOAK — длительный прогон с периодическими срезами
|
||||
# --------------------------------------------------------------------------
|
||||
def soak(args):
|
||||
run_id = f"soak-{int(time.time())}"
|
||||
ns = _ns(run_id)
|
||||
api = API()
|
||||
devices = _create_devices(api, ns, run_id, args.devices)
|
||||
try:
|
||||
counter = Counter()
|
||||
pubs = [DevicePublisher(run_id, ns, d["device_id"], d["username"],
|
||||
d["password"], rate=args.rate,
|
||||
duration=args.duration, qos=args.qos,
|
||||
counter=counter) for d in devices]
|
||||
for p in pubs:
|
||||
p.start()
|
||||
v = Verifier(api, ns, run_id)
|
||||
t0 = now_ms()
|
||||
slices = []
|
||||
while any(p.is_alive() for p in pubs):
|
||||
time.sleep(args.slice_sec)
|
||||
sent = counter.get("sent")
|
||||
rep = v.run_report(devices, 0)
|
||||
slices.append({
|
||||
"elapsed_sec": int((now_ms() - t0) / 1000),
|
||||
"sent": sent,
|
||||
"delivered": rep["delivered"],
|
||||
})
|
||||
print(json.dumps(slices[-1]))
|
||||
for p in pubs:
|
||||
p.join()
|
||||
expected = max(1, int(args.rate * args.duration))
|
||||
rep = v.run_report(devices, expected)
|
||||
rep.update({"scenario": "soak", "ns": ns, "slices": slices})
|
||||
return rep
|
||||
finally:
|
||||
_cleanup(api, ns, devices)
|
||||
Reference in New Issue
Block a user