From 5711e97fe59b7e12c2d8b95e8971398b53cc9a7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Tue, 25 Aug 2026 08:08:14 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20=D0=BC=D0=BE=D0=B4=D1=83=D0=BB=D1=8C=20?= =?UTF-8?q?upload=20=E2=80=94=20=D0=B2=D1=8B=D0=BD=D0=BE=D1=81=20=D1=81?= =?UTF-8?q?=D0=BB=D0=BE=D1=91=D0=B2=201=20=D0=B8=202=20(=D0=B2=D1=8B=D0=B1?= =?UTF-8?q?=D0=BE=D1=80=20=D1=84=D0=B0=D0=B9=D0=BB=D0=BE=D0=B2=20+=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=BA=D0=B0=D1=87=D0=BA=D0=B0=20=D1=87=D0=B5=D1=80?= =?UTF-8?q?=D0=B5=D0=B7=20=D0=92=D0=9C),=20=D1=84=D0=B0=D0=B9=D0=BB-=D0=BD?= =?UTF-8?q?=D0=B0-=D1=84=D1=83=D0=BD=D0=BA=D1=86=D0=B8=D1=8E,=20=D1=82?= =?UTF-8?q?=D0=B5=D1=81=D1=82=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_upload_module.py | 269 +++++++++++++++++++ upload/README.md | 123 +++++++++ upload/__init__.py | 8 + upload/backend/__init__.py | 1 + upload/backend/session/__init__.py | 37 +++ upload/backend/session/add_file.py | 25 ++ upload/backend/session/cancel.py | 27 ++ upload/backend/session/cleanup.py | 11 + upload/backend/session/create_session.py | 23 ++ upload/backend/session/get_files.py | 23 ++ upload/backend/session/state.py | 30 +++ upload/backend/session/store_csv.py | 30 +++ upload/backend/session/store_result.py | 30 +++ upload/backend/session/ttl.py | 34 +++ upload/backend/upload_refs/__init__.py | 20 ++ upload/backend/upload_refs/blueprint.py | 110 ++++++++ upload/backend/upload_refs/config.py | 11 + upload/backend/upload_refs/pull_file.py | 35 +++ upload/backend/upload_refs/safe_name.py | 16 ++ upload/config.example.json | 11 + upload/frontend/package.json | 6 + upload/frontend/table/add_file_with_dedup.js | 47 ++++ upload/frontend/table/esc.js | 5 + upload/frontend/table/est_for_file.js | 8 + upload/frontend/table/fmt_sec.js | 6 + upload/frontend/table/fs.js | 7 + upload/frontend/table/init_upload_table.js | 57 ++++ upload/frontend/table/on_files_change.js | 56 ++++ upload/frontend/table/on_folder_change.js | 58 ++++ upload/frontend/table/proc_row.js | 13 + upload/frontend/table/render.js | 29 ++ upload/frontend/table/render_proc_table.js | 74 +++++ upload/frontend/table/rm_file.js | 19 ++ upload/frontend/table/set_status.js | 6 + upload/frontend/test_upload_frontend.mjs | 213 +++++++++++++++ upload/frontend/upload/put_to_vm.js | 22 ++ upload/frontend/upload/upload_via_vm.js | 58 ++++ upload/frontend/zip/decode_zip_name.js | 22 ++ upload/frontend/zip/dos_to_ms.js | 11 + upload/frontend/zip/inflate_raw.js | 8 + upload/frontend/zip/list_zip_files.js | 22 ++ upload/frontend/zip/parse_zip.js | 50 ++++ 42 files changed, 1671 insertions(+) create mode 100644 tests/test_upload_module.py create mode 100644 upload/README.md create mode 100644 upload/__init__.py create mode 100644 upload/backend/__init__.py create mode 100644 upload/backend/session/__init__.py create mode 100644 upload/backend/session/add_file.py create mode 100644 upload/backend/session/cancel.py create mode 100644 upload/backend/session/cleanup.py create mode 100644 upload/backend/session/create_session.py create mode 100644 upload/backend/session/get_files.py create mode 100644 upload/backend/session/state.py create mode 100644 upload/backend/session/store_csv.py create mode 100644 upload/backend/session/store_result.py create mode 100644 upload/backend/session/ttl.py create mode 100644 upload/backend/upload_refs/__init__.py create mode 100644 upload/backend/upload_refs/blueprint.py create mode 100644 upload/backend/upload_refs/config.py create mode 100644 upload/backend/upload_refs/pull_file.py create mode 100644 upload/backend/upload_refs/safe_name.py create mode 100644 upload/config.example.json create mode 100644 upload/frontend/package.json create mode 100644 upload/frontend/table/add_file_with_dedup.js create mode 100644 upload/frontend/table/esc.js create mode 100644 upload/frontend/table/est_for_file.js create mode 100644 upload/frontend/table/fmt_sec.js create mode 100644 upload/frontend/table/fs.js create mode 100644 upload/frontend/table/init_upload_table.js create mode 100644 upload/frontend/table/on_files_change.js create mode 100644 upload/frontend/table/on_folder_change.js create mode 100644 upload/frontend/table/proc_row.js create mode 100644 upload/frontend/table/render.js create mode 100644 upload/frontend/table/render_proc_table.js create mode 100644 upload/frontend/table/rm_file.js create mode 100644 upload/frontend/table/set_status.js create mode 100644 upload/frontend/test_upload_frontend.mjs create mode 100644 upload/frontend/upload/put_to_vm.js create mode 100644 upload/frontend/upload/upload_via_vm.js create mode 100644 upload/frontend/zip/decode_zip_name.js create mode 100644 upload/frontend/zip/dos_to_ms.js create mode 100644 upload/frontend/zip/inflate_raw.js create mode 100644 upload/frontend/zip/list_zip_files.js create mode 100644 upload/frontend/zip/parse_zip.js diff --git a/tests/test_upload_module.py b/tests/test_upload_module.py new file mode 100644 index 0000000..aa5f82a --- /dev/null +++ b/tests/test_upload_module.py @@ -0,0 +1,269 @@ +"""Тесты переиспользуемого модуля upload (backend): session, safe_name, blueprint. + +Запуск: + python3 tests/test_upload_module.py + pytest tests/test_upload_module.py +""" + +import os +import sys + +# Корень проекта — для импорта пакета upload +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from flask import Flask + +from upload.backend.session import ( + create_session, add_file, get_files, file_count, + store_result, get_result, store_csv, get_csv, + cleanup, request_cancel, get_cancel_event, +) +from upload.backend.upload_refs import safe_name, create_upload_refs_blueprint + +VM_PREFIX = "https://contracts.kube5s.ru/drhider-upload/" + + +# ═══════════════════════════════════════════════════════════════════════════ +# session +# ═══════════════════════════════════════════════════════════════════════════ + +def test_create_session(): + sid = create_session() + assert len(sid) == 32 # uuid4().hex + + +def test_add_file_and_get(): + sid = create_session() + assert add_file(sid, "a.txt", b"hello") is True + assert get_files(sid) == [("a.txt", b"hello")] + assert file_count(sid) == 1 + + +def test_add_file_missing_session(): + assert add_file("nonexistent", "a.txt", b"x") is False + assert get_files("nonexistent") is None + + +def test_session_limit(monkeypatch): + # importlib: `import ...session.add_file as m` затеняется одноимённой функцией + import importlib + add_file_mod = importlib.import_module("upload.backend.session.add_file") + old = add_file_mod.MAX_SESSION_BYTES + add_file_mod.MAX_SESSION_BYTES = 10 + try: + sid = create_session() + assert add_file(sid, "a", b"12345") is True + # суммарно уже 5; +6 = 11 > 10 → False + assert add_file(sid, "b", b"123456") is False + assert file_count(sid) == 1 + finally: + add_file_mod.MAX_SESSION_BYTES = old + + +def test_store_result_and_csv(): + sid = create_session() + assert store_result(sid, b"zip") is True + assert get_result(sid) == b"zip" + assert store_csv(sid, "a,b") is True + assert get_csv(sid) == "a,b" + + +def test_cleanup(): + sid = create_session() + cleanup(sid) + assert get_files(sid) is None + + +def test_cancel(): + sid = create_session() + assert request_cancel(sid) is True + ev = get_cancel_event(sid) + assert ev is not None and ev.is_set() + assert request_cancel("nonexistent") is False + assert get_cancel_event("nonexistent") is None + + +# ═══════════════════════════════════════════════════════════════════════════ +# safe_name +# ═══════════════════════════════════════════════════════════════════════════ + +def test_safe_name(): + assert safe_name("a.txt") == "a.txt" + assert safe_name("dir/sub/a.txt") == "dir/sub/a.txt" + assert safe_name("dir\\sub\\a.txt") == "dir/sub/a.txt" + assert safe_name("../etc/passwd") == "" + assert safe_name("a/../../b") == "" + assert safe_name("") == "" + assert safe_name("./a.txt") == "a.txt" + assert safe_name("/etc/passwd") == "etc/passwd" + + +# ═══════════════════════════════════════════════════════════════════════════ +# blueprint upload_refs +# ═══════════════════════════════════════════════════════════════════════════ + +class _FakeStream: + def __init__(self, payload=b"", status=200, exc=None): + self._payload = payload + self._status = status + self._exc = exc + + def __enter__(self): + if self._exc: + raise self._exc + return self + + def __exit__(self, *a): + return False + + def raise_for_status(self): + if self._status >= 400: + raise RuntimeError("HTTP %d" % self._status) + + def iter_bytes(self): + yield self._payload + + +class _FakeClient: + """Имитация httpx.Client: stream/delete без сети. + + behavior: dict url -> "ok" | {"payload": bytes} | {"exc": Exception} + """ + + def __init__(self, behavior): + self._behavior = behavior + self.deleted = [] + self.streams = [] + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def stream(self, method, url): + self.streams.append((method, url)) + b = self._behavior.get(url) + if isinstance(b, dict) and "payload" in b: + return _FakeStream(payload=b["payload"]) + if isinstance(b, dict) and "exc" in b: + return _FakeStream(exc=b["exc"]) + if b == "ok": + return _FakeStream() + return _FakeStream(status=500) + + def delete(self, url): + self.deleted.append(url) + + +def _make_client(behavior, cfg=None, monkeypatch=None): + import upload.backend.upload_refs.blueprint as bp_mod + monkeypatch.setattr(bp_mod.httpx, "Client", lambda **kw: _FakeClient(behavior)) + app = Flask(__name__) + app.register_blueprint(create_upload_refs_blueprint(cfg or {})) + return app.test_client() + + +def test_upload_refs_no_files(monkeypatch): + c = _make_client({}, monkeypatch=monkeypatch) + rv = c.post("/api/upload_refs", json={"session": "", "files": []}) + assert rv.status_code == 400 + + +def test_upload_refs_ok(monkeypatch): + url = VM_PREFIX + "tok_0" + c = _make_client({url: {"payload": b"hello"}}, monkeypatch=monkeypatch) + rv = c.post("/api/upload_refs", json={ + "session": "", + "files": [{"name": "a.txt", "size": 5, "url": url}], + }) + assert rv.status_code == 200 + data = rv.get_json() + assert data["ok"] is True + assert data["count"] == 1 + files = get_files(data["session"]) + assert files == [("a.txt", b"hello")] + + +def test_upload_refs_ssrf(monkeypatch): + c = _make_client({}, monkeypatch=monkeypatch) + rv = c.post("/api/upload_refs", json={ + "session": "", + "files": [{"name": "evil.txt", "size": 5, "url": "https://evil.example/x"}], + }) + assert rv.status_code == 200 + data = rv.get_json() + assert data["ok"] is True + assert data["count"] == 0 # unsafe URL пропущен + + +def test_upload_refs_too_large(monkeypatch): + url = VM_PREFIX + "tok_0" + c = _make_client({}, monkeypatch=monkeypatch) + rv = c.post("/api/upload_refs", json={ + "session": "", + "files": [{"name": "big.bin", "size": 60 * 1024 * 1024, "url": url}], + }) + assert rv.status_code == 200 + assert rv.get_json()["count"] == 0 # сверх 50МБ → skip+delete + + +def test_upload_refs_session_not_found(monkeypatch): + url = VM_PREFIX + "tok_0" + c = _make_client({url: {"payload": b"x"}}, monkeypatch=monkeypatch) + rv = c.post("/api/upload_refs", json={ + "session": "no-such-session", + "files": [{"name": "a.txt", "size": 1, "url": url}], + }) + assert rv.status_code == 404 + assert rv.get_json()["error"] == "Session not found" + + +def test_upload_refs_pull_failed(monkeypatch): + url = VM_PREFIX + "tok_0" + behavior = {url: {"exc": RuntimeError("dns fail")}} + c = _make_client(behavior, cfg={"pullRetryDelay": 0}, monkeypatch=monkeypatch) + rv = c.post("/api/upload_refs", json={ + "session": "", + "files": [{"name": "a.txt", "size": 1, "url": url}], + }) + assert rv.status_code == 502 # все ретраи провалились + + +# ═══════════════════════════════════════════════════════════════════════════ +# main (запуск без pytest) +# ═══════════════════════════════════════════════════════════════════════════ + +def main(): + """Прогнать все test_* функции без pytest.""" + import inspect + import traceback + fns = [(k, v) for k, v in sorted(globals().items()) if k.startswith("test_")] + mp = _FakeMonkeyPatch() + failed = 0 + for name, fn in fns: + try: + if "monkeypatch" in inspect.signature(fn).parameters: + fn(monkeypatch=mp) + else: + fn() + print("PASS %s" % name) + except Exception: + failed += 1 + print("FAIL %s" % name) + traceback.print_exc() + if failed: + print("%d тестов упало" % failed) + sys.exit(1) + print("Все тесты прошли") + + +class _FakeMonkeyPatch: + """Мини-monkeypatch: setattr, работает до конца вызова (без undo).""" + + def setattr(self, target, name, value): + setattr(target, name, value) + + +if __name__ == "__main__": + main() diff --git a/upload/README.md b/upload/README.md new file mode 100644 index 0000000..1fc450f --- /dev/null +++ b/upload/README.md @@ -0,0 +1,123 @@ +# upload — переиспользуемые слои загрузки через ВМ + +Два самодостаточных слоя для выноса в любой другой проект БЕЗ изменения кода +(меняется только конфиг). Поведение 1:1 с drhider v0.0.75. + +``` +upload/ + frontend/ + zip/ # распаковка ZIP (чистые функции) + table/ # слой 1: выбор файлов/папки/архива, дедуп, статусы, таблица + upload/ # слой 2 (фронт): PUT на ВМ + POST /api/upload_refs + backend/ + upload_refs/ # слой 2 (бэк): Blueprint upload_refs (SSRF, _safe_name, ретраи) + session/ # in-memory сессия с TTL и лимитами + config.example.json +``` + +## Что это + +| Слой | Где | Ответственность | +|---|---|---| +| **1. Выбор файлов** | фронт | таблица, дедуп, раскрытие ZIP, путь, статусы, кнопки | +| **2. Закачка через ВМ** | фронт + бэк | PUT на ВМ-буфер (фронт) → `upload_refs` pull (бэк) → сессия | +| **3. Логика приложения** | — | у каждого приложения своя (обфускация, SSE и т.п.). В модуле её НЕТ | + +Паттерн (зачем ВМ): шлюз managed-кластера рвёт тела >64КБ, egress не ограничен. +Поэтому: браузер → `PUT` на ВМ-буфер → Flask `POST /api/upload_refs` → egress `GET` → сессия. + +--- + +## Подключение фронта + +Подключить ES-модули (`