From aa585220fc87407409e2ffff259eeee9ab26feed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Wed, 26 Aug 2026 20:47:41 +0300 Subject: [PATCH] =?UTF-8?q?refactor:=20=D1=83=D0=B4=D0=B0=D0=BB=D1=91?= =?UTF-8?q?=D0=BD=20=D0=BC=D1=91=D1=80=D1=82=D0=B2=D1=8B=D0=B9=20legacy-?= =?UTF-8?q?=D0=BA=D0=BE=D0=B4=20=D0=B7=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BA?= =?UTF-8?q?=D0=B8=20=D0=BD=D0=B0=D0=BF=D1=80=D1=8F=D0=BC=D1=83=D1=8E=20(/u?= =?UTF-8?q?pload,=20/unzip-upload,=20=5Fcheck=5Fext,=20=5Funzip,=20ALLOWED?= =?UTF-8?q?,=20=D0=BA=D0=BE=D0=BD=D1=81=D1=82=D0=B0=D0=BD=D1=82=D1=8B=20?= =?UTF-8?q?=D1=84=D1=80=D0=BE=D0=BD=D1=82=D0=B0);=20=D0=B7=D0=B0=D0=B3?= =?UTF-8?q?=D1=80=D1=83=D0=B7=D0=BA=D0=B0=20=D1=82=D0=BE=D0=BB=D1=8C=D0=BA?= =?UTF-8?q?=D0=BE=20=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20=D0=92=D0=9C=20(/api/?= =?UTF-8?q?upload=5Frefs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- site/routes/__init__.py | 3 +- site/routes/upload_bp.py | 84 ++--------------------------------- site/static/app.js | 3 -- tests/README.md | 6 +-- tests/test_routes.py | 37 --------------- tests/test_upload_security.py | 50 --------------------- 6 files changed, 7 insertions(+), 176 deletions(-) delete mode 100644 tests/test_upload_security.py diff --git a/site/routes/__init__.py b/site/routes/__init__.py index dd32119..ef458ab 100644 --- a/site/routes/__init__.py +++ b/site/routes/__init__.py @@ -3,7 +3,7 @@ def register_routes(app): import config - from routes.upload_bp import upload_bp, contracts_upload_sink + from routes.upload_bp import contracts_upload_sink from routes.pipeline_bp import pipeline_bp from routes.api_bp import api_bp from routes.prompts_bp import prompts_bp @@ -11,7 +11,6 @@ def register_routes(app): from routes.pages_bp import pages_bp from upload.backend.upload_refs import create_upload_refs_blueprint - app.register_blueprint(upload_bp) app.register_blueprint(pipeline_bp) app.register_blueprint(api_bp) app.register_blueprint(prompts_bp) diff --git a/site/routes/upload_bp.py b/site/routes/upload_bp.py index fba2384..274d213 100644 --- a/site/routes/upload_bp.py +++ b/site/routes/upload_bp.py @@ -1,54 +1,12 @@ -"""Upload blueprint — загрузка, конвертация, распаковка.""" -import io, os, base64, hashlib, zipfile +"""Upload-модуль: конвертация .doc→.docx + хранение/парсинг (sink для VM-буфера).""" +import base64 +import hashlib + import httpx -from flask import Blueprint, request, jsonify from services.parse import parse_file from db import documents import config -upload_bp = Blueprint("upload", __name__) - -ALLOWED = {"pdf", "docx", "doc", "zip"} - - -def _check_ext(filename: str) -> str | None: - ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" - if ext not in ALLOWED: - return f"unsupported format: .{ext} (allowed: {', '.join(sorted(ALLOWED))})" - return None - - -def _unzip(data: bytes): - """Распаковать ZIP → (ok, files, error).""" - MAX_FILES = 500 - MAX_UNCOMPRESSED = 500 * 1024 * 1024 # 500 MB - files = [] - total = 0 - try: - with zipfile.ZipFile(io.BytesIO(data)) as zf: - if len(zf.namelist()) > MAX_FILES: - return False, None, f"too many files in ZIP (max {MAX_FILES})" - for info in zf.infolist(): - if info.is_dir(): - continue - name = os.path.basename(info.filename) - if not name or ".." in name or "/" in name or "\\" in name: - continue - raw = zf.read(info) - total += len(raw) - if total > MAX_UNCOMPRESSED: - return False, None, "total uncompressed size exceeds 500 MB" - ext = name.rsplit(".", 1)[-1].lower() if "." in name else "" - files.append({ - "filename": name, - "ext": ext, - "size": len(raw), - "data_b64": base64.b64encode(raw).decode(), - }) - except zipfile.BadZipFile: - return False, None, "invalid ZIP archive" - return True, files, None - def _convert(filename: str, data: bytes) -> bytes: """.doc → .docx через внешний libreoffice-сервис. Возвращает docx-байты.""" @@ -104,28 +62,6 @@ def _store_and_parse(filename: str, data: bytes, batch_id, contract_id, zip_sour return {"ok": True, "doc_id": doc["id"], "contract_id": contract_id, "parsed": parsed} -@upload_bp.route("/upload", methods=["POST"]) -def upload(): - """Загрузка одного файла + авто-парсинг → БД (прямой multipart).""" - f = request.files.get("files") - if not f: - return jsonify(ok=False, error="no file"), 400 - - err = _check_ext(f.filename) - if err: - return jsonify(ok=False, error=err), 400 - - data = f.read() - batch_id = request.form.get("batch_id") - zip_source = request.form.get("zip_source") - contract_id = request.form.get("contract_id") - - result = _store_and_parse(f.filename, data, batch_id, contract_id, zip_source, f.content_type or "application/octet-stream") - if not result["ok"]: - return jsonify(ok=result["ok"], error=result.get("error"), doc_id=result.get("doc_id")), 200 - return jsonify(ok=True, doc_id=result["doc_id"], contract_id=result["contract_id"], parsed=result["parsed"]) - - def contracts_upload_sink(name, content, batch_id=None, contract_id=None, zip_source=None): """Sink для переиспользуемого модуля upload: вставить файл в documents + авто-парсинг. @@ -137,15 +73,3 @@ def contracts_upload_sink(name, content, batch_id=None, contract_id=None, zip_so content = _convert(name, content) name = name[:-4] + ".docx" return _store_and_parse(name, content, batch_id, contract_id, zip_source) - - -@upload_bp.route("/unzip-upload", methods=["POST"]) -def unzip_upload(): - """Распаковать ZIP → список файлов (base64 для фронтенда, прямой multipart).""" - f = request.files.get("files") - if not f: - return jsonify(ok=False, error="no file"), 400 - ok, files, err = _unzip(f.read()) - if not ok: - return jsonify(ok=False, error=err), 400 - return jsonify(ok=True, files=files) diff --git a/site/static/app.js b/site/static/app.js index 0f78b11..4a4ce3a 100644 --- a/site/static/app.js +++ b/site/static/app.js @@ -1,9 +1,6 @@ // ⛔ НЕ МЕНЯТЬ БЕЗ РАЗРЕШЕНИЯ НАЕЛЯ ⛔ // Contracts App v2.0 — всё на Flask, ВМ больше нет var VM_API = ''; -var UPLOAD_URL = '/upload'; -var CONVERT_URL = '/convert-doc'; -var UNZIP_URL = '/unzip-upload'; // ВМ-буфер загрузки (паттерн drhider): браузер кладёт файл сюда (WebDAV, мимо шлюза), // бэк сам тянет его по /api/upload_refs. Origin должен быть в CORS на nginx ВМ. var VM_UPLOAD_URL = 'https://contracts.kube5s.ru/contracts-upload/'; diff --git a/tests/README.md b/tests/README.md index a20e3af..2ee8d2f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,14 +27,12 @@ pytest установлен в venv: `/home/naeel/nubes/contracts/.venv/bin/pyth | `test_spec_events.py` | `db/spec_events.py`: `apply_ops` (ADD/UPDATE/DELETE/UNRESOLVED/unknown), `clear_current`, `reset`, `_hash`, `get_next_seq` | | `test_spec_current.py` | `db/spec_current.py`: `list_by_contract` (порядок), `get_elements_json` | | `test_process_pipeline.py` | `services/process.py`: `run_pipeline` с Fake LLM — **full_replace не дублирует строки**, **трансляция target_id→target_hash**, `_elements_to_text` | -| `test_routes.py` | HTTP-эндпоинты (Flask test client): `/health`, `/api/spec-current`, `/upload`, `/unzip-upload`, `/api/groups` | -| `test_upload_security.py` | `routes/upload_bp.py`: `_check_ext` (допустимые/недопустимые), `_unzip` (path traversal нейтрализуется, битый zip) | +| `test_routes.py` | HTTP-эндпоинты (Flask test client): `/health`, `/api/spec-current`, `/api/groups` | ## Ключевые проверки - `test_full_replace_no_duplicates` — два `full_replace` подряд → в `spec_current` 3 строки, а не 6; история `spec_events` (6 событий) сохранена. - `test_update_applies` — `target_id: "r1"` транслируется в `target_hash` → UPDATE применяется (status `applied`), а не уходит в UNRESOLVED. -- `test_path_traversal_neutralized` — `../etc/passwd` внутри ZIP → имя файла `passwd` без пути. ## Изоляция @@ -56,7 +54,7 @@ pytest -m "not load" -q # быстрые юнит-тесты без на | `load/test_load_pipeline.py` | массовый `run_pipeline`: N контрактов × M допников (Fake LLM) — проверка, что `full_replace` не дублирует строки в масштабе | | `load/test_load_apply_ops.py` | массовые `apply_ops`: N ADD → N UPDATE → N DELETE, целостность `spec_events` | | `load/test_load_concurrency.py` | конкурентная запись в SQLite (WAL + `busy_timeout`), проверка отсутствия потери данных | -| `load/stress_http.py` | standalone HTTP-стресс (`/upload`, `/health`) — параллельные запросы, RPS/ошибки/таймауты | +| `load/stress_http.py` | standalone HTTP-стресс (`/health`; загрузка `/upload` удалена — реальный путь `/api/upload_refs`) | Масштаб через переменные окружения (дефолты — большие): diff --git a/tests/test_routes.py b/tests/test_routes.py index 31cd79a..8be456a 100644 --- a/tests/test_routes.py +++ b/tests/test_routes.py @@ -1,7 +1,4 @@ """Интеграционные тесты HTTP-эндпоинтов (Flask test client).""" -import io -import zipfile - import pytest @@ -45,40 +42,6 @@ class TestSpecCurrent: assert data["rows"] == [] -class TestUpload: - def test_no_file(self, client): - r = client.post("/upload", data={}) - assert r.status_code == 400 - - def test_unsupported_ext(self, client): - data = {"files": (io.BytesIO(b"x"), "test.exe")} - r = client.post("/upload", data=data, content_type="multipart/form-data") - assert r.status_code == 400 - - def test_docx_creates_document(self, client): - data = {"files": (io.BytesIO(b"not a real docx"), "test.docx")} - r = client.post("/upload", data=data, content_type="multipart/form-data") - assert r.status_code == 200 - j = r.get_json() - assert j["ok"] is True - assert j["doc_id"] - - -class TestUnzipUpload: - def test_unzip(self, client): - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w") as zf: - zf.writestr("a.docx", b"hello") - zf.writestr("b.pdf", b"world") - buf.seek(0) - data = {"files": (buf, "test.zip")} - r = client.post("/unzip-upload", data=data, content_type="multipart/form-data") - assert r.status_code == 200 - j = r.get_json() - assert j["ok"] is True - assert len(j["files"]) == 2 - - class TestGroups: def test_missing_batch(self, client): r = client.get("/api/groups") diff --git a/tests/test_upload_security.py b/tests/test_upload_security.py deleted file mode 100644 index 6b57ddc..0000000 --- a/tests/test_upload_security.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Тесты безопасности routes/upload_bp.py — чистые функции, без HTTP.""" -import io -import zipfile - -from routes.upload_bp import _check_ext, _unzip - - -class TestCheckExt: - def test_allowed(self): - assert _check_ext("a.pdf") is None - assert _check_ext("a.docx") is None - assert _check_ext("a.doc") is None - assert _check_ext("a.zip") is None - - def test_rejected(self): - assert _check_ext("a.exe") is not None - assert _check_ext("a.txt") is not None - assert _check_ext("noext") is not None - - def test_uppercase(self): - assert _check_ext("A.PDF") is None - - -class TestUnzip: - def _zip(self, entries): - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w") as zf: - for name, content in entries.items(): - zf.writestr(name, content) - return buf.getvalue() - - def test_valid(self): - ok, files, err = _unzip(self._zip({"a.docx": b"hello", "b.pdf": b"world"})) - assert ok is True - assert len(files) == 2 - assert files[0]["filename"] == "a.docx" - assert files[0]["data_b64"] - - def test_path_traversal_neutralized(self): - ok, files, err = _unzip(self._zip({"../etc/passwd": b"evil"})) - assert ok is True - # basename убирает каталог — имя файла не содержит путь - assert files[0]["filename"] == "passwd" - assert ".." not in files[0]["filename"] - assert "/" not in files[0]["filename"] - - def test_invalid_zip(self): - ok, files, err = _unzip(b"not a zip at all") - assert ok is False - assert err == "invalid ZIP archive"