From a05a9691a065595c8956af7be08fe437b2e83538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Sun, 28 Jun 2026 09:20:36 +0400 Subject: [PATCH] =?UTF-8?q?feat(=D0=A41):=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80?= =?UTF-8?q?=D0=B0=D0=BA=D1=82=D1=8B=20=D1=81=D1=82=D0=B0=D0=B4=D0=B8=D0=B9?= =?UTF-8?q?=20+=20parse=5Fmultipart=20+=20store=5Fdocument?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/contracts.py | 84 ++++++++++++++++++++++++++++++ deploy/services/upload.py | 105 ++++++++++++++++++++++++++------------ 2 files changed, 156 insertions(+), 33 deletions(-) create mode 100644 deploy/contracts.py diff --git a/deploy/contracts.py b/deploy/contracts.py new file mode 100644 index 0000000..9916403 --- /dev/null +++ b/deploy/contracts.py @@ -0,0 +1,84 @@ +"""Контракты стадий пайплайна — frozen dataclass на выходах, dict на входах.""" +from dataclasses import dataclass, field +from typing import Optional +from datetime import date + + +@dataclass(frozen=True) +class ParseResult: + """Выход parse.py — результат парсинга документа.""" + status: str # 'parsed' | 'error' + element_count: int # количество извлечённых элементов + elements: list # [{type, text/rows}, ...] + error: Optional[str] = None # сообщение об ошибке (если status='error') + + @classmethod + def from_row(cls, row: dict): + """Из строки БД (documents).""" + return cls( + status=row.get("status", "error"), + element_count=0, # вычисляется отдельно + elements=row.get("elements_json", []), + error=row.get("error_message"), + ) + + +@dataclass(frozen=True) +class ClassifyResult: + """Выход classify.py — результат классификации документа.""" + doc_type: str # 'contract' | 'supplement' | 'specification' | 'garbage' | 'other' + own_number: Optional[str] = None + parent_number: Optional[str] = None + doc_date: Optional[str] = None + counterparty: Optional[str] = None + + @classmethod + def from_llm(cls, raw: dict): + """Из сырого ответа LLM.""" + return cls( + doc_type=raw.get("doc_type", "other"), + own_number=raw.get("own_number") or None, + parent_number=raw.get("parent_number") or None, + doc_date=raw.get("doc_date") or None, + counterparty=raw.get("counterparty") or None, + ) + + +@dataclass(frozen=True) +class GroupingResult: + """Выход grouping.py — результат группировки по контрактам.""" + contract_number: str # нормализованный номер договора + counterparty: Optional[str] = None + documents: list = field(default_factory=list) # [{doc_id, filename, doc_type, ...}] + unresolved: list = field(default_factory=list) # документы без группы + + @classmethod + def from_row(cls, row: dict): + """Из строки БД или вычисленной группы.""" + return cls( + contract_number=row.get("contract_number", ""), + counterparty=row.get("counterparty"), + documents=row.get("documents", []), + unresolved=row.get("unresolved", []), + ) + + +@dataclass(frozen=True) +class CompareOp: + """Одна операция сравнения (ADD/UPDATE/DELETE/UNRESOLVED).""" + action: str # 'ADD' | 'UPDATE' | 'DELETE' | 'UNRESOLVED' + new_row: Optional[dict] = None + new_values: Optional[dict] = None + target_hash: Optional[str] = None + comment: str = "" + + @classmethod + def from_llm(cls, raw: dict): + """Из сырого ответа LLM.""" + return cls( + action=raw.get("action", "UNRESOLVED"), + new_row=raw.get("new_row") or None, + new_values=raw.get("new_values") or None, + target_hash=raw.get("target_hash") or raw.get("target_id"), + comment=raw.get("comment", ""), + ) diff --git a/deploy/services/upload.py b/deploy/services/upload.py index 852dfb4..017bcc9 100644 --- a/deploy/services/upload.py +++ b/deploy/services/upload.py @@ -6,30 +6,24 @@ from .parse import parse_file MAX_FILE_SIZE = 100 * 1024 * 1024 # 100 MB -def handle_upload(rfile, content_type, content_length): - """Parse multipart upload, store in DB, return result dict.""" - # Validate content_length - try: - cl = int(content_length) - except (TypeError, ValueError): - return {"ok": False, "error": "invalid content-length"} - if cl <= 0: - return {"ok": False, "error": "empty request"} - if cl > MAX_FILE_SIZE: - return {"ok": False, "error": f"file too large (max {MAX_FILE_SIZE // 1024 // 1024}MB)"} - - body = rfile.read(cl) +# ── Чистые функции (тестируемы без HTTP) ────────────────────── +def parse_multipart(raw_bytes: bytes, content_type: str) -> dict: + """Разобрать multipart/form-data → {filename, file_data, contract_id, batch_id, zip_source}. + Тестируется без HTTP — передать bytes. + """ environ = { "REQUEST_METHOD": "POST", "CONTENT_TYPE": content_type, - "CONTENT_LENGTH": str(content_length), + "CONTENT_LENGTH": str(len(raw_bytes)), } - fs = cgi.FieldStorage(fp=io.BytesIO(body), environ=environ, keep_blank_values=True) + fs = cgi.FieldStorage(fp=io.BytesIO(raw_bytes), environ=environ, keep_blank_values=True) filename = None file_data = None contract_id = "" + batch_id = None + zip_source = None if "files" in fs: item = fs["files"] @@ -37,37 +31,50 @@ def handle_upload(rfile, content_type, content_length): item = item[0] filename = os.path.basename(item.filename) if item.filename else None if filename and (".." in filename or "/" in filename or "\\" in filename): - return {"ok": False, "error": "invalid filename"} + raise ValueError("invalid filename") file_data = item.file.read() if hasattr(item, "file") else item.value if isinstance(file_data, str): file_data = file_data.encode("utf-8") if "contract_id" in fs: - contract_id = fs.getfirst("contract_id", "") - # Validate UUID - if contract_id: + cid = fs.getfirst("contract_id", "") + if cid: try: - uuid.UUID(contract_id) + uuid.UUID(cid) + contract_id = cid except (ValueError, AttributeError): - contract_id = "" + pass - batch_id = fs.getfirst("batch_id", None) if "batch_id" in fs else None - # Validate UUID - if batch_id: - try: - uuid.UUID(batch_id) - except (ValueError, AttributeError): - batch_id = None + if "batch_id" in fs: + bid = fs.getfirst("batch_id", None) + if bid: + try: + uuid.UUID(bid) + batch_id = bid + except (ValueError, AttributeError): + pass - # zip_source — имя родительского ZIP-архива (для визуальной группировки) - zip_source = None if "zip_source" in fs: raw_zip = fs.getfirst("zip_source", "") if raw_zip: - zip_source = os.path.basename(raw_zip)[:255] # защита от path traversal + лимит + zip_source = os.path.basename(raw_zip)[:255] + return { + "filename": filename, + "file_data": file_data, + "contract_id": contract_id or "", + "batch_id": batch_id, + "zip_source": zip_source, + } + + +def store_document(filename: str, file_data: bytes, contract_id: str = "", + batch_id: str = None, zip_source: str = None) -> dict: + """Сохранить документ в БД + распарсить. Возвращает {ok, doc_id, contract_id, parsed, ...}. + Тестируется с реальной или замоканной БД. + """ if not filename or not file_data: - return {"ok": False, "error": "no file in request"} + return {"ok": False, "error": "no file"} mime = _mime_for(filename) b64 = base64.b64encode(file_data).decode() @@ -76,7 +83,7 @@ def handle_upload(rfile, content_type, content_length): try: supplements.delete_by_document(contract_id, filename) except Exception: - pass # old record may not exist or FK issue — proceed with insert + pass doc = documents.insert(filename, mime, b64, batch_id=batch_id, zip_source=zip_source) @@ -107,6 +114,38 @@ def handle_upload(rfile, content_type, content_length): } +# ── HTTP-handler (вызывает parse_multipart + store_document) ─── + +def handle_upload(rfile, content_type, content_length): + """HTTP-handler: читает rfile → parse_multipart → store_document.""" + try: + cl = int(content_length) + except (TypeError, ValueError): + return {"ok": False, "error": "invalid content-length"} + if cl <= 0: + return {"ok": False, "error": "empty request"} + if cl > MAX_FILE_SIZE: + return {"ok": False, "error": f"file too large (max {MAX_FILE_SIZE // 1024 // 1024}MB)"} + + raw_bytes = rfile.read(cl) + + try: + parts = parse_multipart(raw_bytes, content_type) + except ValueError as e: + return {"ok": False, "error": str(e)} + + if not parts["filename"] or not parts["file_data"]: + return {"ok": False, "error": "no file in request"} + + return store_document( + filename=parts["filename"], + file_data=parts["file_data"], + contract_id=parts["contract_id"], + batch_id=parts["batch_id"], + zip_source=parts["zip_source"], + ) + + def _mime_for(filename): ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" mime_map = {