435 lines
16 KiB
Python
435 lines
16 KiB
Python
# scenarios.py — нагрузочные сценарии.
|
||
#
|
||
# Каждый сценарий: prepare → run → verify → report (dict) → cleanup.
|
||
# Все устройства создаются самим тестом и помечены run_id; cleanup удаляет
|
||
# только их (если IOT_CLEANUP=1).
|
||
import json
|
||
import threading
|
||
import time
|
||
|
||
from .common import API, CLEANUP, Counter, NS_PREFIX, latency_stats, now_ms
|
||
from .publisher import DevicePublisher
|
||
from .verifier import Verifier
|
||
|
||
|
||
def _ns(run_id):
|
||
return f"{NS_PREFIX}_{run_id}"
|
||
|
||
|
||
def _create_devices(api, ns, run_id, count, device_prefix="dev"):
|
||
devices = []
|
||
for i in range(count):
|
||
name = f"{device_prefix}{i}"
|
||
d = api.create_device(ns, name, f"{device_prefix}-{i}")
|
||
# POST возвращает устройство БЕЗ пароля — пароль только в GET по имени.
|
||
d = api.get_device(ns, name)
|
||
devices.append({
|
||
"name": name,
|
||
"device_id": d["device_id"],
|
||
"username": d["mqtt_username"],
|
||
"password": d["mqtt_password"],
|
||
})
|
||
return devices
|
||
|
||
|
||
def _cleanup(api, ns, devices):
|
||
if not CLEANUP:
|
||
return
|
||
for d in devices:
|
||
try:
|
||
api.delete_device(ns, d["name"])
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _run_publishers(run_id, ns, devices, **kw):
|
||
"""Запускает паблишеры, ждёт завершения, возвращает счётчики."""
|
||
counter = Counter()
|
||
pubs = [DevicePublisher(run_id, ns, d["device_id"], d["username"],
|
||
d["password"], counter=counter, **kw)
|
||
for d in devices]
|
||
for p in pubs:
|
||
p.start()
|
||
for p in pubs:
|
||
p.join()
|
||
return counter.snapshot()
|
||
|
||
|
||
def _wait_delivery(api, ns, run_id, devices, expected_total, timeout=120):
|
||
"""Ждёт, пока доедет expected_total сообщений (или таймаут)."""
|
||
v = Verifier(api, ns, run_id)
|
||
t0 = time.monotonic()
|
||
while time.monotonic() - t0 < timeout:
|
||
per = sum(v.device_report(d["device_id"], 0)["delivered"]
|
||
for d in devices)
|
||
rep = v.run_report(devices, expected_per_device=0)
|
||
if rep["delivered"] >= expected_total:
|
||
return rep, time.monotonic() - t0
|
||
time.sleep(5)
|
||
return v.run_report(devices, expected_per_device=0), time.monotonic() - t0
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 1. BASELINE — равномерная нагрузка
|
||
# --------------------------------------------------------------------------
|
||
def baseline(args):
|
||
run_id = f"base-{int(time.time())}"
|
||
ns = _ns(run_id)
|
||
api = API()
|
||
devices = _create_devices(api, ns, run_id, args.devices)
|
||
try:
|
||
expected_per_device = max(1, int(args.rate * args.duration))
|
||
t0 = now_ms()
|
||
counters = _run_publishers(
|
||
run_id, ns, devices, rate=args.rate, duration=args.duration,
|
||
qos=args.qos, payload_size=args.payload_size,
|
||
reconnect_every=args.reconnect_every)
|
||
send_ms = now_ms() - t0
|
||
time.sleep(10) # consumer long-poll до ~20с
|
||
rep = Verifier(api, ns, run_id).run_report(
|
||
devices, expected_per_device)
|
||
rep.update({
|
||
"scenario": "baseline",
|
||
"ns": ns,
|
||
"send_ms": send_ms,
|
||
"publisher": counters,
|
||
"rate_rps": round(counters.get("sent", 0) / max(1, send_ms / 1000), 2),
|
||
})
|
||
return rep
|
||
finally:
|
||
_cleanup(api, ns, devices)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 2. BURST — всплеск
|
||
# --------------------------------------------------------------------------
|
||
def burst(args):
|
||
run_id = f"burst-{int(time.time())}"
|
||
ns = _ns(run_id)
|
||
api = API()
|
||
devices = _create_devices(api, ns, run_id, args.devices)
|
||
try:
|
||
quiet = 20
|
||
spike = args.duration
|
||
low = _run_publishers(run_id + "-l", ns, devices,
|
||
rate=args.rate, duration=quiet, qos=args.qos)
|
||
high = _run_publishers(run_id + "-h", ns, devices,
|
||
rate=args.rate * args.burst_mult,
|
||
duration=spike, qos=args.qos)
|
||
tail = _run_publishers(run_id + "-t", ns, devices,
|
||
rate=args.rate, duration=quiet, qos=args.qos)
|
||
# Фазы различаются суффиксом run_id — верифицируем фазу "-h" отдельно.
|
||
high_sent = high.get("sent", 0)
|
||
expected_per_device = max(1, high_sent // max(1, len(devices)))
|
||
rep_high = Verifier(api, ns, run_id + "-h").run_report(
|
||
devices, expected_per_device)
|
||
# recovery: сколько времени после конца всплеска очередь отдала всё
|
||
_, recovery_s = _wait_delivery(
|
||
api, ns, run_id + "-h", devices, high_sent, timeout=300)
|
||
return {
|
||
"scenario": "burst",
|
||
"ns": ns,
|
||
"quiet_phase": low,
|
||
"spike_phase": high,
|
||
"tail_phase": tail,
|
||
"sent_high_total": high_sent,
|
||
"recovery_sec": round(recovery_s, 1),
|
||
"high": rep_high,
|
||
}
|
||
finally:
|
||
_cleanup(api, ns, devices)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 3. LARGE_PAYLOAD — крупные сообщения
|
||
# Верификация через API невозможна: строка с большим payload в ответе >15 КБ,
|
||
# а шлюз платформы рвёт такие ответы (баг MSS/MTU, тикет Nubes). Доставку
|
||
# проверять по логам монолита: 'consumer: telemetry saved'.
|
||
# --------------------------------------------------------------------------
|
||
def large_payload(args):
|
||
run_id = f"big-{int(time.time())}"
|
||
ns = _ns(run_id)
|
||
api = API()
|
||
devices = _create_devices(api, ns, run_id, min(args.devices, 10))
|
||
try:
|
||
counters = _run_publishers(
|
||
run_id, ns, devices, rate=args.rate, duration=args.duration,
|
||
qos=args.qos, payload_size=args.payload_size)
|
||
return {
|
||
"scenario": "large_payload",
|
||
"ns": ns,
|
||
"payload_size": args.payload_size,
|
||
"publisher": counters,
|
||
"note": "доставку смотреть в логах монолита (telemetry saved); "
|
||
"API-верификация невозможна из-за бага шлюза (>15KB ответ)",
|
||
}
|
||
finally:
|
||
_cleanup(api, ns, devices)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 4. RECONNECT_STORM — массовые переподключения
|
||
# --------------------------------------------------------------------------
|
||
def reconnect_storm(args):
|
||
run_id = f"rc-{int(time.time())}"
|
||
ns = _ns(run_id)
|
||
api = API()
|
||
devices = _create_devices(api, ns, run_id, args.devices)
|
||
try:
|
||
expected = max(1, int(args.rate * args.duration))
|
||
counters = _run_publishers(
|
||
run_id, ns, devices, rate=args.rate, duration=args.duration,
|
||
qos=1, reconnect_every=args.reconnect_every)
|
||
time.sleep(10)
|
||
rep = Verifier(api, ns, run_id).run_report(devices, expected)
|
||
rep.update({
|
||
"scenario": "reconnect_storm",
|
||
"ns": ns,
|
||
"publisher": counters,
|
||
"reconnect_every_sec": args.reconnect_every,
|
||
})
|
||
return rep
|
||
finally:
|
||
_cleanup(api, ns, devices)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 5. MULTITENANT — много namespace, задержка первого сообщения
|
||
# --------------------------------------------------------------------------
|
||
def multitenant(args):
|
||
run_id = f"mt-{int(time.time())}"
|
||
api = API()
|
||
devices_per_tenant = args.devices_per_tenant
|
||
all_devices = []
|
||
tenant_devices = {}
|
||
for t in range(args.tenants):
|
||
ns = _ns(f"{run_id}-{t}")
|
||
devs = _create_devices(api, ns, run_id, devices_per_tenant,
|
||
device_prefix=f"t{t}")
|
||
tenant_devices[ns] = devs
|
||
all_devices.extend(devs)
|
||
try:
|
||
expected = max(1, int(args.rate * args.duration))
|
||
t0 = now_ms()
|
||
for ns, devs in tenant_devices.items():
|
||
_run_publishers(run_id, ns, devs, rate=args.rate,
|
||
duration=args.duration, qos=args.qos)
|
||
time.sleep(15)
|
||
first_lats = []
|
||
for ns, devs in tenant_devices.items():
|
||
v = Verifier(api, ns, run_id)
|
||
for d in devs:
|
||
r = v.device_report(d["device_id"], expected)
|
||
if r["first_latency_ms"] is not None:
|
||
first_lats.append(r["first_latency_ms"])
|
||
return {
|
||
"scenario": "multitenant",
|
||
"tenants": args.tenants,
|
||
"devices_per_tenant": devices_per_tenant,
|
||
"total_devices": len(all_devices),
|
||
"elapsed_ms": now_ms() - t0,
|
||
"first_msg_latency_ms": latency_stats(first_lats),
|
||
}
|
||
finally:
|
||
for ns, devs in tenant_devices.items():
|
||
_cleanup(api, ns, devs)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 6. ACL_VIOLATION — публикация в чужой топик
|
||
# --------------------------------------------------------------------------
|
||
def acl_violation(args):
|
||
run_id = f"acl-{int(time.time())}"
|
||
ns = _ns(run_id)
|
||
api = API()
|
||
devices = _create_devices(api, ns, run_id, 2)
|
||
try:
|
||
victim_ns = _ns(f"{run_id}-victim")
|
||
victim = _create_devices(api, victim_ns, run_id, 1)[0]
|
||
attacker = devices[0]
|
||
foreign_topic = f"{victim_ns}/telemetry/{victim['device_id']}"
|
||
counter = Counter()
|
||
p = DevicePublisher(run_id, ns, attacker["device_id"],
|
||
attacker["username"], attacker["password"],
|
||
rate=1, duration=10, qos=0, counter=counter,
|
||
extra_topic=foreign_topic, publish_foreign=True)
|
||
p.start()
|
||
p.join()
|
||
return {
|
||
"scenario": "acl_violation",
|
||
"ns": ns,
|
||
"publisher": counter.snapshot(),
|
||
"expect": "foreign_denied=все, foreign_accepted=0 "
|
||
"(EMQX рвёт сессию: deny_action=disconnect)",
|
||
}
|
||
finally:
|
||
_cleanup(api, ns, devices)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 7. AUTH_NEG — отказ на неверные креды
|
||
# --------------------------------------------------------------------------
|
||
def auth_neg(args):
|
||
import paho.mqtt.client as mqtt
|
||
|
||
from .common import WS_HOST, WS_PATH, WS_PORT, ws_tls_opts
|
||
|
||
def attempt(client_id, username, password):
|
||
res = {}
|
||
c = mqtt.Client(client_id=client_id, transport="websockets")
|
||
c.ws_set_options(path=WS_PATH)
|
||
c.username_pw_set(username, password)
|
||
ws_tls_opts(c)
|
||
|
||
def on_connect(cl, ud, flags, rc, props=None):
|
||
res["rc"] = rc
|
||
c.on_connect = on_connect
|
||
try:
|
||
c.connect(WS_HOST, WS_PORT, 15)
|
||
c.loop_start()
|
||
time.sleep(2)
|
||
c.loop_stop()
|
||
except Exception as e:
|
||
res["exc"] = type(e).__name__
|
||
return res
|
||
|
||
results = {
|
||
"wrong_password": attempt("lt-neg-pass", "test_dev-001", "wrong-123"),
|
||
"unknown_user": attempt("lt-neg-user", "nosuch_user", "whatever"),
|
||
}
|
||
return {"scenario": "auth_neg",
|
||
"expect": "wrong_password rc=4, unknown_user rc=5",
|
||
"results": results}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 8. API_CRUD — нагрузка на CRUD устройств
|
||
# --------------------------------------------------------------------------
|
||
def api_crud(args):
|
||
run_id = f"api-{int(time.time())}"
|
||
ns = _ns(run_id)
|
||
api = API()
|
||
lat = {"POST": [], "GET": [], "DELETE": []}
|
||
lock = threading.Lock()
|
||
errors = Counter()
|
||
|
||
def worker(wid):
|
||
for i in range(args.iterations):
|
||
name = f"w{wid}-{i}"
|
||
for op, fn in (
|
||
("POST", lambda: api.create_device(ns, name, f"w{wid}-{i}")),
|
||
("GET", lambda: api.get_device(ns, name)),
|
||
("DELETE", lambda: api.delete_device(ns, name)),
|
||
):
|
||
t0 = now_ms()
|
||
try:
|
||
fn()
|
||
with lock:
|
||
lat[op].append(now_ms() - t0)
|
||
except Exception:
|
||
errors.inc(op)
|
||
|
||
threads = [threading.Thread(target=worker, args=(w,))
|
||
for w in range(args.threads)]
|
||
for t in threads:
|
||
t.start()
|
||
for t in threads:
|
||
t.join()
|
||
return {
|
||
"scenario": "api_crud",
|
||
"ns": ns,
|
||
"threads": args.threads,
|
||
"iterations_per_thread": args.iterations,
|
||
"POST": latency_stats(lat["POST"]),
|
||
"GET": latency_stats(lat["GET"]),
|
||
"DELETE": latency_stats(lat["DELETE"]),
|
||
"errors": errors.snapshot(),
|
||
}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 9. TELEMETRY_QUERY — нагрузка на чтение телеметрии
|
||
# --------------------------------------------------------------------------
|
||
def telemetry_query(args):
|
||
run_id = f"tq-{int(time.time())}"
|
||
ns = _ns(run_id)
|
||
api = API()
|
||
devices = _create_devices(api, ns, run_id, max(1, args.devices))
|
||
try:
|
||
# сидируем данные
|
||
_run_publishers(run_id, ns, devices, rate=5, duration=20, qos=0)
|
||
time.sleep(10)
|
||
lat = []
|
||
lock = threading.Lock()
|
||
errors = Counter()
|
||
|
||
def worker(wid):
|
||
for _ in range(args.iterations):
|
||
t0 = now_ms()
|
||
try:
|
||
api.telemetry(ns, limit=args.query_limit)
|
||
with lock:
|
||
lat.append(now_ms() - t0)
|
||
except Exception:
|
||
errors.inc("query")
|
||
|
||
threads = [threading.Thread(target=worker, args=(w,))
|
||
for w in range(args.threads)]
|
||
t0 = now_ms()
|
||
for t in threads:
|
||
t.start()
|
||
for t in threads:
|
||
t.join()
|
||
elapsed = max(1, (now_ms() - t0) / 1000)
|
||
return {
|
||
"scenario": "telemetry_query",
|
||
"ns": ns,
|
||
"threads": args.threads,
|
||
"iterations": args.iterations,
|
||
"query_limit": args.query_limit,
|
||
"req_per_sec": round(len(lat) / elapsed, 2),
|
||
"latency": latency_stats(lat),
|
||
"errors": errors.snapshot(),
|
||
}
|
||
finally:
|
||
_cleanup(api, ns, devices)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 10. SOAK — длительный прогон с периодическими срезами
|
||
# --------------------------------------------------------------------------
|
||
def soak(args):
|
||
run_id = f"soak-{int(time.time())}"
|
||
ns = _ns(run_id)
|
||
api = API()
|
||
devices = _create_devices(api, ns, run_id, args.devices)
|
||
try:
|
||
counter = Counter()
|
||
pubs = [DevicePublisher(run_id, ns, d["device_id"], d["username"],
|
||
d["password"], rate=args.rate,
|
||
duration=args.duration, qos=args.qos,
|
||
counter=counter) for d in devices]
|
||
for p in pubs:
|
||
p.start()
|
||
v = Verifier(api, ns, run_id)
|
||
t0 = now_ms()
|
||
slices = []
|
||
while any(p.is_alive() for p in pubs):
|
||
time.sleep(args.slice_sec)
|
||
sent = counter.get("sent")
|
||
rep = v.run_report(devices, 0)
|
||
slices.append({
|
||
"elapsed_sec": int((now_ms() - t0) / 1000),
|
||
"sent": sent,
|
||
"delivered": rep["delivered"],
|
||
})
|
||
print(json.dumps(slices[-1]))
|
||
for p in pubs:
|
||
p.join()
|
||
expected = max(1, int(args.rate * args.duration))
|
||
rep = v.run_report(devices, expected)
|
||
rep.update({"scenario": "soak", "ns": ns, "slices": slices})
|
||
return rep
|
||
finally:
|
||
_cleanup(api, ns, devices)
|