refactor: services/ → compare/ + drhider/ (разделение Сверки и Хайдера)

deploy/compare/classify.py

@@ -0,0 +1,267 @@
+"""
+Classify service — LLM-based document classification.
+
+Архитектурное решение (Opus):
+- Отдельный сервис, не встроен в upload. Upload быстрый (0.5с), classify — медленный (2-10с/файл).
+- ThreadPoolExecutor(max_workers=4) — параллельная классификация с ограничением конкурентности,
+  чтобы не положить api.aillm.ru при 2000 файлах.
+- Умная выжимка (_smart_extract): header ~1500 симв + regex-хиты по маркерам (договор/№/соглашение)
+  из всего документа. Экономия токенов в 5-10 раз при сохранении точности.
+- Двухпроходная архитектура: LLM извлекает строки (тип/номер/дата/контрагент),
+  Python в grouping.py нормализует и группирует детерминированно.
+"""
+import json, re, os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import httpx
+from db import documents as db_docs
+from llm_prompt import build_classify_prompt
+
+log = __import__("logging").getLogger(__name__)
+
+# Лимит одновременных запросов к LLM API
+# Увеличивать осторожно — api.aillm.ru может троттлить
+MAX_WORKERS = 4
+
+LLM_URL = "https://api.aillm.ru/v1/chat/completions"
+LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "")
+LLM_MODEL = "gpt-oss-120b"
+
+# Ленивый singleton — обратная совместимость
+_classify_llm = None
+
+
+def _get_classify_client():
+    global _classify_llm
+    if _classify_llm is None:
+        from .llm_client import HttpxLLMClient
+        _classify_llm = HttpxLLMClient(url=LLM_URL, key=LLM_KEY, model=LLM_MODEL, max_tokens=1000, timeout=60)
+    return _classify_llm
+
+# ── Garbage filter (Stage 1: filename regex) ────────────────────
+_GARBAGE_FILENAME_RE = re.compile(
+    r'(сч[её]т|акт|плат[её]ж|УПД|сверк|инвойс|invoice|payment|act)',
+    re.IGNORECASE,
+)
+
+# ── Garbage filter (Stage 2: header keywords) ───────────────────
+_GARBAGE_HEADER_MARKERS = [
+    'СЧЕТ-ФАКТУРА', 'СЧЕТ НА ОПЛАТУ', 'АКТ СВЕРКИ',
+    'АКТ ОКАЗАННЫХ УСЛУГ', 'АКТ ВЫПОЛНЕННЫХ РАБОТ',
+    'ПЛАТЁЖНОЕ ПОРУЧЕНИЕ', 'УНИВЕРСАЛЬНЫЙ ПЕРЕДАТОЧНЫЙ',
+    'УПД', 'ПЛАТЕЖНОЕ ПОРУЧЕНИЕ',
+]
+
+
+def _is_garbage_by_filename(filename: str) -> bool:
+    """Stage 1: regex по имени файла — быстро, 0 токенов."""
+    return bool(_GARBAGE_FILENAME_RE.search(filename))
+
+
+def _is_garbage_by_header(text: str) -> bool:
+    """Stage 2: ключевые слова в первых 2KB текста — быстро, 0 токенов."""
+    header = text[:2000].upper()
+    return any(marker in header for marker in _GARBAGE_HEADER_MARKERS)
+
+
+def _call_llm_classify(header_text, llm_client=None):
+    """
+    Прямой вызов LLM для классификации ОДНОГО документа.
+    Возвращает (parsed_dict, raw_text, needed_fix).
+    llm_client: LLMClient (optional). Default — HttpxLLMClient.
+    """
+    if llm_client is None:
+        llm_client = _get_classify_client()
+
+    prompt, _ = build_classify_prompt(header_text)
+    raw_text = llm_client.complete(prompt)
+    parsed, needed_fix = _safe_json_parse(raw_text)
+    return parsed, raw_text, needed_fix
+
+
+def classify_batch(batch_id, llm_client=None, repo=None):
+    """
+    Классифицировать все документы в batch.
+    Сбрасывает статус на 'pending' для всех перед началом.
+    Параллельно (ThreadPoolExecutor) обрабатывает до MAX_WORKERS документов.
+    Возвращает {ok, total, done, failed, garbage, json_fix_rate}.
+    llm_client: LLMClient (optional, default — HttpxLLMClient)
+    repo: Repository (optional, default — direct db.* calls)
+    """
+    _db = repo if repo else db_docs
+    _llm = llm_client if llm_client else _get_classify_client()
+
+    # Сбросить статус — allow re-classify after file changes
+    _db.reset_classify_status(batch_id)
+    pending = _db.list_pending(batch_id)
+    if not pending:
+        return {"ok": False, "error": "no pending documents"}
+
+    total = len(pending)
+    done = 0
+    failed = 0
+    garbage = 0
+    json_fixes = 0
+    json_total = 0
+    type_counts = {}  # doc_type → count for batch summary
+
+    def _classify_one(doc):
+        """Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
+        nonlocal garbage, json_fixes, json_total, type_counts
+        try:
+            # Stage 1: garbage by filename (0 tokens)
+            if _is_garbage_by_filename(doc["filename"]):
+                _db.set_classify_garbage(doc["id"], "filename_regex")
+                garbage += 1
+                return True
+
+            # Stage 2: garbage by header keywords (0 tokens)
+            text = _smart_extract(doc["elements_json"])
+            if _is_garbage_by_header(text):
+                _db.set_classify_garbage(doc["id"], "header_keywords")
+                garbage += 1
+                return True
+
+            # Stage 3: LLM classification (only for remaining)
+            _db.set_classify_processing(doc["id"])  # crash recovery marker
+            result, raw, needed_fix = _call_llm_classify(text, _llm)
+            json_total += 1
+            if needed_fix:
+                json_fixes += 1
+            dtype = result.get("doc_type", "other")
+            type_counts[dtype] = type_counts.get(dtype, 0) + 1
+            _db.set_classification(
+                doc["id"],
+                result.get("doc_type", "other"),
+                result.get("own_number"),
+                result.get("parent_number"),
+                result.get("doc_date"),
+                result.get("counterparty"),
+                classify_raw=raw,
+                classify_input=text,
+            )
+            return True
+        except Exception as e:
+            _db.set_classify_failed(doc["id"], str(e))
+            return False
+
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
+        futures = {pool.submit(_classify_one, d): d for d in pending}
+        for f in as_completed(futures):
+            if f.result():
+                done += 1
+            else:
+                failed += 1
+
+    return {"ok": True, "total": total, "done": done, "failed": failed,
+            "garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3),
+            "types": type_counts, "summary": f"{total} total, {done} classified, {failed} failed, {garbage} garbage"}
+
+
+def _safe_json_parse(raw):
+    """Parse LLM response, fixing common JSON errors.
+    Returns (parsed_dict, needed_fix: bool)."""
+    if not raw:
+        raise ValueError("empty LLM response")
+
+    text = raw.strip()
+    # Strip markdown
+    if "```json" in text:
+        text = text.split("```json")[1].split("```")[0].strip()
+    elif "```" in text:
+        text = text.split("```")[1].split("```")[0].strip()
+
+    # Remove non-JSON prefix/suffix (LLM chatter)
+    brace_start = text.find("{")
+    brace_end = text.rfind("}")
+    if brace_start >= 0 and brace_end > brace_start:
+        text = text[brace_start:brace_end + 1]
+
+    # Try strict parse
+    try:
+        return json.loads(text), False
+    except json.JSONDecodeError:
+        pass
+
+    import re as _re
+    # Collapse multiline
+    text = _re.sub(r"\n\s*", " ", text)
+    # Remove trailing commas
+    text = _re.sub(r",\s*}", "}", text)
+    text = _re.sub(r",\s*]", "]", text)
+
+    # Try again
+    try:
+        return json.loads(text), True
+    except json.JSONDecodeError:
+        pass
+
+    # Aggressive: try adding missing closing quotes/braces
+    text = text.rstrip()
+    if not text.endswith("}"):
+        # Count unclosed quotes
+        in_string = False
+        for i, ch in enumerate(text):
+            if ch == '"' and (i == 0 or text[i-1] != "\\"):
+                in_string = not in_string
+        if in_string:
+            text += '"'
+        text += "}"
+
+    return json.loads(text), True
+
+
+def _smart_extract(elements_json):
+    """
+    Умная выжимка текста для классификации (решение Q3 от Opus).
+
+    Вместо отправки всего документа (дорого) или только header (теряет зарытые номера),
+    используется гибрид:
+    1. Первые ~1500 симв (титул, преамбула, стороны)
+    2. Regex-хиты по маркерам «договор|№|соглашение|приложение|спецификация»
+       из ВСЕГО документа
+    3. Дедупликация, лимит 10 строк, склейка → ~3000 симв на вход LLM
+
+    Это покрывает и титульную зону, и зарытые ссылки в середине документа.
+    """
+    if not elements_json:
+        return ""
+
+    if isinstance(elements_json, str):
+        try:
+            elements = json.loads(elements_json)
+        except json.JSONDecodeError:
+            return elements_json[:2000]
+    elif isinstance(elements_json, list):
+        elements = elements_json
+    else:
+        return str(elements_json)[:2000]
+
+    # Build full text
+    lines = []
+    for el in elements:
+        if isinstance(el, dict):
+            t = el.get("type") or el.get("TYPE", "")
+            if t == "paragraph":
+                txt = el.get("text") or el.get("TEXT", "")
+                if txt:
+                    lines.append(txt)
+            elif t == "table":
+                rows = el.get("rows") or el.get("ROWS", [])
+                for row in rows:
+                    lines.append(" | ".join(str(c) for c in row))
+
+    full_text = "\n".join(lines)
+
+    # Header: first ~1500 chars
+    header = full_text[:1500]
+
+    # Marker lines: grep for key patterns
+    markers = re.findall(
+        r'.{0,200}(?:договор|№|соглашен|приложен|специф|контрагент|заказчик|арендатор).{0,200}',
+        full_text, re.IGNORECASE,
+    )
+    unique_markers = list(dict.fromkeys(markers))[:10]
+
+    combined = header + "\n---\n" + "\n".join(unique_markers)
+    return combined[:3000]

deploy/compare/grouping.py

@@ -0,0 +1,160 @@
+"""
+Grouping service — match classified documents into contract groups.
+
+Архитектурное решение (Opus, Q4 + Q6):
+- Двухпроходный гибрид: LLM извлекает строки (classify.py), Python нормализует и группирует.
+- Нормализация номеров: uppercase + только буквы/цифры.
+  "МЭС-123-2024" == "МЭС 123/2024" после нормализации.
+- Группировка на бэкенде (не на фронте): Python regex/unicode надёжнее JS.
+- apply_groups(): создаёт contracts + supplements с авто-порядком по дате.
+"""
+import re
+from db import documents as db_docs
+from db import contracts as db_contracts
+from db import supplements as db_supplements
+
+
+def normalize_number(num):
+    """
+    Нормализация номера договора для сравнения.
+    Убирает всё кроме букв и цифр, приводит к uppercase.
+    Пример: "МЭС-123-2024" → "МЭС1232024", "МЭС 123/2024" → "МЭС1232024".
+    """
+    if not num:
+        return ""
+    return re.sub(r"[^A-Z0-9А-Я]", "", num.upper())
+
+
+def group_documents(batch_id):
+    """
+    Сгруппировать классифицированные документы по контрактам.
+
+    Алгоритм:
+    1. Отделить contract от supplement/specification
+    2. Каждый contract → якорь группы
+    3. Для каждого supplement: найти contract по parent_number (нормализованный)
+    4. Оставшиеся supplement/spec → виртуальные группы по parent_number/own_number
+    5. Совсем без номеров → группа "__unresolved__"
+    6. Внутри группы сортировка по doc_date
+
+    Возвращает {ok, groups: [{contract_number, counterparty, documents: [...]}], total_docs}.
+    """
+    docs = db_docs.list_by_batch(batch_id)
+    classified = [d for d in docs if d.get("classify_status") == "classified"]
+
+    # Separate contracts and supplements
+    contracts_list = []
+    supplements_list = []
+
+    for d in classified:
+        if d.get("doc_type") == "contract":
+            contracts_list.append(d)
+        else:
+            supplements_list.append(d)
+
+    groups = []
+
+    # Each contract becomes a group
+    for c in contracts_list:
+        group = {
+            "contract_number": c.get("own_number") or c.get("filename", ""),
+            "counterparty": c.get("counterparty") or "",
+            "documents": [c],
+        }
+        # Find supplements matching this contract
+        c_norm = normalize_number(c.get("own_number"))
+        for s in supplements_list:
+            parent = normalize_number(s.get("parent_number") or "")
+            own = normalize_number(s.get("own_number") or "")
+            if parent == c_norm or own == c_norm:
+                if s not in group["documents"]:
+                    group["documents"].append(s)
+
+        # Sort by date
+        group["documents"].sort(key=lambda x: x.get("doc_date") or "")
+
+        # Remove matched supplements from the pool
+        for s in group["documents"]:
+            if s in supplements_list:
+                supplements_list.remove(s)
+
+        groups.append(group)
+
+    # ── Virtual groups: unmatched supplements grouped by number ──────────
+    # Group remaining supplements/specs by normalized parent_number (priority) or own_number
+    virtual = {}
+    for s in supplements_list:
+        num = normalize_number(s.get("parent_number") or s.get("own_number") or "")
+        if not num:
+            continue  # no number → stays in supplements_list for unresolved
+        if num not in virtual:
+            # Use counterparty from first doc in group
+            cp = s.get("counterparty") or ""
+            virtual[num] = {
+                "contract_number": s.get("parent_number") or s.get("own_number") or "?",
+                "counterparty": cp,
+                "documents": [],
+            }
+        virtual[num]["documents"].append(s)
+        # Update counterparty if current doc has a better one
+        if not virtual[num]["counterparty"] and s.get("counterparty"):
+            virtual[num]["counterparty"] = s.get("counterparty")
+
+    for vnum, vgroup in virtual.items():
+        vgroup["documents"].sort(key=lambda x: x.get("doc_date") or "")
+        groups.append(vgroup)
+        # Remove grouped docs from supplements_list
+        for s in vgroup["documents"]:
+            supplements_list.remove(s)
+
+    # ── Unresolved: everything left (no number, failed, pending, other) ──
+    unmatched = [s for s in supplements_list]  # remaining after virtual grouping
+    unmatched += [d for d in docs if d.get("classify_status") != "classified"]
+
+    if unmatched:
+        groups.append({
+            "contract_number": "__unresolved__",
+            "counterparty": "",
+            "documents": unmatched,
+        })
+
+    return {"ok": True, "groups": groups, "total_docs": len(docs)}
+
+
+def apply_groups(batch_id, groups_data):
+    """
+    Применить подтверждённые группы: создать contracts + supplements.
+
+    Вызывается из POST /api/apply-groups.
+    Для каждой группы (кроме __unresolved__):
+    1. Создать запись в contracts (number, client)
+    2. Для каждого документа создать supplement (type='initial' для первого, 'additional' для остальных)
+    3. Порядок supplements соответствует порядку документов в группе (сортировка по дате уже сделана)
+
+    Возвращает {ok, created: количество созданных supplements}.
+    """
+    created = 0
+    contract_ids = []
+    for g in groups_data:
+        contract_number = g.get("contract_number", "")
+        if contract_number == "__unresolved__":
+            continue
+        counterparty = g.get("counterparty", "")
+        docs = g.get("documents", [])
+
+        try:
+            c = db_contracts.insert(contract_number, counterparty)
+            contract_id = c["id"]
+            contract_ids.append(contract_id)
+
+            for i, d in enumerate(docs):
+                doc_id = d.get("id")
+                if not doc_id:
+                    continue
+                supp_type = "initial" if i == 0 else "additional"
+                db_supplements.insert(contract_id, doc_id, supp_type)
+                created += 1
+        except Exception as e:
+            return {"ok": False, "error": f"apply_groups failed at {contract_number}: {e}"}
+
+    return {"ok": True, "created": created, "contract_ids": contract_ids}

deploy/compare/llm.py

@@ -0,0 +1,35 @@
+"""LLM service — call LLM API with optional DI."""
+import os, json
+
+LLM_URL = "https://api.aillm.ru/v1/chat/completions"
+LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "")
+LLM_MODEL = "gpt-oss-120b"
+
+# Ленивый singleton — обратная совместимость
+_llm_client = None
+
+
+def _get_default_client():
+    global _llm_client
+    if _llm_client is None:
+        from .llm_client import HttpxLLMClient
+        _llm_client = HttpxLLMClient(url=LLM_URL, key=LLM_KEY, model=LLM_MODEL)
+    return _llm_client
+
+
+def call_llm(current_spec, doc_text, build_prompt_fn, llm_client=None):
+    """Call LLM. Returns (parsed_result, prompt_id).
+    llm_client: LLMClient (optional). Default — HttpxLLMClient (prod).
+    """
+    if llm_client is None:
+        llm_client = _get_default_client()
+
+    prompt, prompt_id = build_prompt_fn(current_spec, doc_text)
+    raw_text = llm_client.complete(prompt)
+
+    json_text = raw_text
+    if "```json" in json_text:
+        json_text = json_text.split("```json")[1].split("```")[0]
+    elif "```" in json_text:
+        json_text = json_text.split("```")[1].split("```")[0]
+    return json.loads(json_text.strip()), prompt_id

deploy/compare/llm_client.py

@@ -0,0 +1,74 @@
+"""LLM Client — протокол + httpx-реализация + fake для тестов.
+
+План decoupling Ф2:
+- LLMClient.complete(prompt) -> str — единственный метод
+- Парсинг JSON остаётся у вызывающего (call_llm / _call_llm_classify)
+- HttpxLLMClient — продакшен
+- FakeLLMClient — тесты (возвращает сохранённые ответы)
+"""
+from typing import Protocol
+
+
+class LLMClient(Protocol):
+    """Протокол LLM-клиента. Один метод — сырой вызов."""
+    def complete(self, prompt: str) -> str:
+        """Отправить prompt в LLM, вернуть raw_text."""
+        ...
+
+
+class HttpxLLMClient:
+    """Продакшен-клиент через httpx → api.aillm.ru."""
+
+    def __init__(self, url: str, key: str, model: str = "gpt-oss-120b",
+                 max_tokens: int = 8000, temperature: float = 0.1, timeout: int = 120):
+        self.url = url
+        self.key = key
+        self.model = model
+        self.max_tokens = max_tokens
+        self.temperature = temperature
+        self.timeout = timeout
+
+    def complete(self, prompt: str) -> str:
+        import httpx
+        payload = {
+            "model": self.model,
+            "messages": [{"role": "user", "content": prompt}],
+            "max_tokens": self.max_tokens,
+            "temperature": self.temperature,
+        }
+        with httpx.Client(http2=True, timeout=self.timeout, verify=True) as client:
+            resp = client.post(
+                self.url,
+                json=payload,
+                headers={
+                    "Authorization": f"Bearer {self.key}",
+                    "Content-Type": "application/json",
+                },
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        return data["choices"][0]["message"]["content"]
+
+
+class FakeLLMClient:
+    """Тестовый клиент — возвращает сохранённые ответы из словаря."""
+
+    def __init__(self, responses: dict = None):
+        self.responses = responses or {}
+        self.calls: list[str] = []  # история вызовов для проверок
+
+    def add(self, prompt_key: str, response: str):
+        """Зарегистрировать ответ для конкретного prompt_key."""
+        self.responses[prompt_key] = response
+
+    def complete(self, prompt: str) -> str:
+        self.calls.append(prompt)
+        # Ищем точное совпадение
+        if prompt in self.responses:
+            return self.responses[prompt]
+        # Ищем по частичному ключу
+        for key, resp in self.responses.items():
+            if key in prompt:
+                return resp
+        # Fallback
+        return self.responses.get("default", '{"error": "no response registered"}')

deploy/compare/metrics.py

@@ -0,0 +1,88 @@
+"""Metrics — quality signals without golden dataset.
+
+Checks that work immediately:
+- Arithmetic: sum == price * qty (free signal of LLM/data errors)
+- JSON fix rate: how often _safe_json_parse has to repair LLM output
+"""
+from decimal import Decimal, InvalidOperation
+
+
+def check_arithmetic(ops: list) -> list[dict]:
+    """Check sum == price * qty for ADD/UPDATE operations.
+    Returns list of mismatches: [{action, name, price, qty, expected_sum, actual_sum, diff}]
+    """
+    mismatches = []
+    for op in ops:
+        action = op.get("action", "")
+        if action not in ("ADD", "UPDATE"):
+            continue
+
+        row = op.get("new_row") or op.get("new_values") or {}
+        price = _to_decimal(row.get("price"))
+        qty = _to_decimal(row.get("qty"))
+        actual_sum = _to_decimal(row.get("sum"))
+
+        if price is None or qty is None or actual_sum is None:
+            continue  # can't check without all three
+
+        expected = price * qty
+        if expected != actual_sum:
+            mismatches.append({
+                "action": action,
+                "name": row.get("name", "")[:100],
+                "price": float(price),
+                "qty": float(qty),
+                "expected_sum": float(expected),
+                "actual_sum": float(actual_sum),
+                "diff": float(actual_sum - expected),
+            })
+    return mismatches
+
+
+class ClassifyMetrics:
+    """Track _safe_json_parse fix rate per batch."""
+    def __init__(self):
+        self.total = 0
+        self.fixes = 0  # how many times JSON needed repair
+
+    def record(self, needed_fix: bool):
+        self.total += 1
+        if needed_fix:
+            self.fixes += 1
+
+    @property
+    def fix_rate(self) -> float:
+        return self.fixes / self.total if self.total else 0.0
+
+    def summary(self) -> dict:
+        return {
+            "total_classifications": self.total,
+            "json_fixes": self.fixes,
+            "json_fix_rate": round(self.fix_rate, 3),
+        }
+
+
+def _to_decimal(val) -> Decimal | None:
+    """Safe conversion to Decimal with number normalization."""
+    if val is None or val == "":
+        return None
+    try:
+        s = str(val).replace("\xa0", "").replace(" ", "").replace(",", ".")
+        return Decimal(s)
+    except (InvalidOperation, ValueError):
+        return None
+
+
+def normalize_date(ds: str) -> str | None:
+    """Normalize date to YYYY-MM-DD. Handles DD.MM.YYYY, YYYY-MM-DD, etc."""
+    if not ds:
+        return None
+    import re
+    # DD.MM.YYYY → YYYY-MM-DD
+    m = re.match(r"(\d{2})\.(\d{2})\.(\d{4})", ds)
+    if m:
+        return f"{m.group(3)}-{m.group(2)}-{m.group(1)}"
+    # YYYY-MM-DD — already canonical
+    if re.match(r"\d{4}-\d{2}-\d{2}", ds):
+        return ds
+    return ds  # return as-is if unrecognized format

deploy/compare/parse.py

@@ -0,0 +1,87 @@
+"""Parse service — PDF (pdfplumber), DOCX (python-docx), plain text fallback."""
+import io
+
+
+def parse_file(filename, data):
+    """Parse file bytes → {status, elements, element_count}."""
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    try:
+        if ext == "pdf":
+            return _parse_pdf(data)
+        elif ext == "docx":
+            return _parse_docx(data)
+        elif ext == "doc":
+            return _parse_docx(data)  # python-docx handles most .doc
+        else:
+            return _parse_text(data)
+    except Exception as e:
+        return {"status": "error", "error": str(e), "element_count": 0}
+
+
+def _parse_pdf(data):
+    """Parse PDF with pdfplumber — preserves table structure (unlike PyPDF2)."""
+    import pdfplumber
+    elements = []
+    with pdfplumber.open(io.BytesIO(data)) as pdf:
+        for page in pdf.pages:
+            # Tables first — preserves column structure critical for specs
+            tables = page.extract_tables()
+            for table in tables:
+                if table:
+                    rows = []
+                    for row in table:
+                        if row:
+                            cells = [str(cell or "").strip() for cell in row]
+                            if any(cells):
+                                rows.append(cells)
+                    if rows:
+                        elements.append({"type": "table", "rows": rows})
+            # Remaining text as paragraphs
+            text = page.extract_text()
+            if text:
+                for line in text.split("\n"):
+                    line = line.strip()
+                    if line:
+                        elements.append({"type": "paragraph", "text": line, "style": ""})
+    return {"status": "parsed", "element_count": len(elements), "elements": elements}
+
+
+def _parse_docx(data):
+    import docx
+    doc = docx.Document(io.BytesIO(data))
+    elements = []
+
+    for block in doc.element.body:
+        tag = block.tag.split("}")[-1] if "}" in block.tag else block.tag
+        if tag == "p":
+            text = _extract_paragraph_text(block)
+            if text:
+                elements.append({"type": "paragraph", "text": text, "style": ""})
+        elif tag == "tbl":
+            rows = []
+            for tr in block.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tr"):
+                cells = []
+                for tc in tr.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tc"):
+                    cell_text = "".join(t.text or "" for t in tc.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t"))
+                    cells.append(cell_text.strip())
+                if cells:
+                    rows.append(cells)
+            if rows:
+                elements.append({"type": "table", "rows": rows})
+
+    return {"status": "parsed", "element_count": len(elements), "elements": elements}
+
+
+def _extract_paragraph_text(p_elem):
+    texts = []
+    for t in p_elem.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t"):
+        if t.text:
+            texts.append(t.text)
+    return "".join(texts).strip()
+
+
+def _parse_text(data):
+    text = data.decode("utf-8", errors="ignore")[:5000]
+    lines = [l.strip() for l in text.split("\n") if l.strip()]
+    return {"status": "parsed", "element_count": len(lines),
+            "elements": [{"type": "paragraph", "text": l, "style": ""} for l in lines]}

deploy/compare/process.py

@@ -0,0 +1,140 @@
+"""Process service — SSE pipeline: reset → supplements → LLM → apply."""
+import json, time, threading
+from db import supplements, spec_current, spec_events
+from .metrics import check_arithmetic
+
+
+def run_pipeline(contract_id, order_ids, sse_send, build_prompt_fn):
+    """Run full SSE comparison pipeline. sse_send is a callback(dict)."""
+    t0 = time.time()
+
+    # 0. Reset
+    spec_events.reset(contract_id)
+
+    # 1. Get supplements with parsed documents
+    supps = supplements.list_by_contract(contract_id)
+    if order_ids:
+        order_list = [x.strip() for x in order_ids.split(",") if x.strip()]
+        order_map = {oid: i for i, oid in enumerate(order_list)}
+        supps.sort(key=lambda s: order_map.get(s["id"], 999999))
+    else:
+        # Sort by doc_date (from classification), fallback to created_at
+        supps.sort(key=lambda s: (s.get("doc_date") or "9999-99-99", s.get("created_at", "")))
+
+    if not supps:
+        sse_send({"type": "error", "message": "Нет распарсенных файлов"})
+        return
+
+    from .llm import call_llm
+
+    for s in supps:
+        sid = s["id"]
+
+        # Current spec
+        cur = spec_current.list_by_contract(contract_id)
+        current_spec = []
+        for r in cur:
+            current_spec.append({
+                "hash": r["name_hash"],
+                "name": r["name"],
+                "price": float(r["price"]) if r.get("price") is not None else None,
+                "qty": float(r["qty"]) if r.get("qty") is not None else None,
+                "sum": float(r["sum"]) if r.get("sum") is not None else None,
+                "date_start": r["date_start"],
+            })
+
+        # Get elements_json
+        ej = spec_current.get_elements_json(s["document_id"])
+        if not ej:
+            continue
+
+        if isinstance(ej, dict) and "Value" in ej:
+            ej = ej["Value"]
+        if isinstance(ej, str):
+            elements = json.loads(ej)
+        elif isinstance(ej, list):
+            elements = ej
+        else:
+            elements = []
+
+        elements = [{k.lower(): v for k, v in el.items()} for el in elements]
+        doc_text = _elements_to_text(elements)
+
+        sse_send({"type": "extract_start", "supplement_id": sid, "filename": s["filename"]})
+
+        # LLM call in thread with keepalive
+        t1 = time.time()
+        done = threading.Event()
+        result = [None]
+        error = [None]
+
+        def do_llm():
+            try:
+                result[0] = call_llm(current_spec, doc_text, build_prompt_fn)
+            except Exception as e:
+                error[0] = str(e)
+            finally:
+                done.set()
+
+        t = threading.Thread(target=do_llm)
+        t.start()
+
+        while not done.wait(15):
+            sse_send({"type": "keepalive"})
+
+        t.join()
+        elapsed = round(time.time() - t1, 1)
+
+        if error[0]:
+            sse_send({"type": "extract_error", "supplement_id": sid,
+                       "filename": s["filename"], "error": error[0], "time_s": elapsed})
+            continue
+
+        llm_result, prompt_id = result[0]
+        ops = llm_result.get("ops", [])
+        mode = llm_result.get("mode", "partial")
+
+        # Enrich ops
+        id_map = {f"r{i+1}": r["hash"] for i, r in enumerate(current_spec)}
+        spec_names = {r["hash"]: r["name"] for r in current_spec}
+        for op in ops:
+            tid = op.get("target_id")
+            if tid and tid in id_map:
+                op["target_hash"] = id_map[tid]
+            th = op.get("target_hash")
+            if th and th in spec_names and not op.get("new_row", {}).get("name"):
+                op.setdefault("new_row", {})["name"] = spec_names[th]
+
+        sse_send({"type": "llm_done", "supplement_id": sid, "filename": s["filename"],
+                   "ops_count": len(ops), "mode": mode, "time_s": elapsed})
+
+        # Apply events
+        summary = spec_events.apply_ops(
+            contract_id, sid, s.get("document_id", ""), ops, prompt_id, llm_result
+        )
+        # Arithmetic quality check (free signal, no golden dataset needed)
+        arith_mismatches = check_arithmetic(ops)
+        if arith_mismatches:
+            summary["arithmetic_mismatches"] = len(arith_mismatches)
+        sse_send({"type": "applied", "supplement_id": sid, "summary": summary, "ops": ops})
+
+    total_time = round(time.time() - t0, 1)
+    sse_send({"type": "done", "total_time_s": total_time})
+
+
+def _elements_to_text(elements):
+    lines = []
+    for el in elements:
+        if el.get("type") == "paragraph":
+            prefix = f"[{el['style']}] " if el.get("style") else ""
+            lines.append(prefix + el.get("text", ""))
+        elif el.get("type") == "table":
+            rows = el.get("rows", [])
+            if rows:
+                ncols = len(rows[0])
+                lines.append(f"--- Таблица ({len(rows)}×{ncols}) ---")
+                for row in rows:
+                    cells = [str(c).replace("\n", " ").replace("|", "\\|") for c in row[:ncols]]
+                    lines.append("| " + " | ".join(cells) + " |")
+                lines.append("")
+    return "\n".join(lines)

deploy/compare/unzip.py

@@ -0,0 +1,73 @@
+"""Unzip service — extract files from ZIP archive (no DB, no parse)."""
+import zipfile, io, base64, os
+
+MAX_FILES = 500
+MAX_UNCOMPRESSED = 500 * 1024 * 1024  # 500 MB total
+MAX_RATIO = 100  # max compression ratio (uncompressed/compressed)
+
+
+def _sanitize_zip_name(name: str) -> str | None:
+    """Decode cp437→utf8, strip path traversal, return safe name or None."""
+    # Decode cp437 filenames (common in ZIPs from Windows)
+    try:
+        name = name.encode("cp437").decode("utf-8", errors="replace")
+    except (UnicodeDecodeError, UnicodeEncodeError):
+        pass
+    # Strip directory prefixes and path traversal
+    name = os.path.basename(name)
+    if not name or name.endswith("/") or ".." in name or "/" in name or "\\" in name:
+        return None
+    return name
+
+
+def handle_unzip(rfile, content_length):
+    """Parse ZIP upload, return list of extracted files as base64."""
+    body = rfile.read(content_length)
+
+    # Find file data in raw multipart
+    idx = body.find(b"\r\n\r\n")
+    if idx < 0:
+        return {"ok": False, "error": "invalid multipart"}
+
+    raw = body[idx + 4:]
+    zip_start = raw.find(b"PK\x03\x04")
+    if zip_start < 0:
+        return {"ok": False, "error": "not a ZIP file"}
+
+    zip_data = raw[zip_start:]
+
+    files = []
+    total_uncompressed = 0
+    with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
+        namelist = zf.namelist()
+        if len(namelist) > MAX_FILES:
+            return {"ok": False, "error": f"too many files in ZIP (max {MAX_FILES})"}
+
+        for info in zf.infolist():
+            if info.is_dir():
+                continue
+
+            # ZIP bomb: compression ratio check
+            if info.compress_size > 0:
+                ratio = info.file_size / info.compress_size
+                if ratio > MAX_RATIO:
+                    return {"ok": False, "error": f"suspicious compression ratio ({ratio:.0f}:1) — possible ZIP bomb"}
+
+            name = _sanitize_zip_name(info.filename)
+            if not name:
+                continue  # skip unsafe names
+
+            data = zf.read(info)
+            total_uncompressed += len(data)
+            if total_uncompressed > MAX_UNCOMPRESSED:
+                return {"ok": False, "error": f"uncompressed size exceeds {MAX_UNCOMPRESSED // 1024 // 1024}MB"}
+
+            ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
+            files.append({
+                "filename": name,
+                "data_b64": base64.b64encode(data).decode(),
+                "ext": ext,
+                "size": len(data),
+            })
+
+    return {"ok": True, "files": files}

deploy/compare/upload.py

@@ -0,0 +1,184 @@
+"""Upload service — accept file, store to DB, parse."""
+import uuid, base64, json, io, cgi, os
+from db import documents, contracts, supplements
+from .parse import parse_file
+
+MAX_FILE_SIZE = 100 * 1024 * 1024  # 100 MB
+ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "zip"}
+
+
+def _check_format(filename: str) -> str | None:
+    """Validate file extension. Returns error string or None."""
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    if ext not in ALLOWED_EXTENSIONS:
+        return f"unsupported format: .{ext} (allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))})"
+    return None
+
+
+# ── Чистые функции (тестируемы без HTTP) ──────────────────────
+
+def parse_multipart(raw_bytes: bytes, content_type: str) -> dict:
+    """Разобрать multipart/form-data → {filename, file_data, contract_id, batch_id, zip_source}.
+    Тестируется без HTTP — передать bytes.
+    """
+    environ = {
+        "REQUEST_METHOD": "POST",
+        "CONTENT_TYPE": content_type,
+        "CONTENT_LENGTH": str(len(raw_bytes)),
+    }
+    fs = cgi.FieldStorage(fp=io.BytesIO(raw_bytes), environ=environ, keep_blank_values=True)
+
+    filename = None
+    file_data = None
+    contract_id = ""
+    batch_id = None
+    zip_source = None
+
+    if "files" in fs:
+        item = fs["files"]
+        if isinstance(item, list):
+            item = item[0]
+        raw_filename = item.filename
+        if raw_filename and (".." in raw_filename or "/" in raw_filename or "\\" in raw_filename):
+            raise ValueError("invalid filename")
+        filename = os.path.basename(raw_filename) if raw_filename else None
+        file_data = item.file.read() if hasattr(item, "file") else item.value
+        if isinstance(file_data, str):
+            file_data = file_data.encode("utf-8")
+
+    if "contract_id" in fs:
+        cid = fs.getfirst("contract_id", "")
+        if cid:
+            try:
+                uuid.UUID(cid)
+                contract_id = cid
+            except (ValueError, AttributeError):
+                pass
+
+    if "batch_id" in fs:
+        bid = fs.getfirst("batch_id", None)
+        if bid:
+            try:
+                uuid.UUID(bid)
+                batch_id = bid
+            except (ValueError, AttributeError):
+                pass
+
+    if "zip_source" in fs:
+        raw_zip = fs.getfirst("zip_source", "")
+        if raw_zip:
+            zip_source = os.path.basename(raw_zip)[:255]
+
+    return {
+        "filename": filename,
+        "file_data": file_data,
+        "contract_id": contract_id or "",
+        "batch_id": batch_id,
+        "zip_source": zip_source,
+    }
+
+
+def store_document(filename: str, file_data: bytes, contract_id: str = "",
+                   batch_id: str = None, zip_source: str = None) -> dict:
+    """Сохранить документ в БД + распарсить. Возвращает {ok, doc_id, contract_id, parsed, ...}.
+    Тестируется с реальной или замоканной БД.
+    """
+    if not filename or not file_data:
+        return {"ok": False, "error": "no file"}
+
+    fmt_err = _check_format(filename)
+    if fmt_err:
+        return {"ok": False, "error": fmt_err}
+
+    import hashlib
+    content_hash = hashlib.sha256(file_data).hexdigest()
+
+    # Dedup: same content in same batch → skip
+    if batch_id:
+        existing = documents.get_by_hash(batch_id, content_hash)
+        if existing:
+            return {"ok": True, "duplicate_of": existing["id"], "filename": filename}
+
+    mime = _mime_for(filename)
+    b64 = base64.b64encode(file_data).decode()
+
+    if contract_id:
+        try:
+            supplements.delete_by_document(contract_id, filename)
+        except Exception as e:
+            import logging
+            logging.getLogger(__name__).warning("delete_by_document failed: %s", e)
+
+    doc = documents.insert(filename, mime, b64, batch_id=batch_id, zip_source=zip_source,
+                           content_hash=content_hash)
+
+    parsed = parse_file(filename, file_data)
+    if parsed and parsed.get("status") == "parsed":
+        documents.set_parsed(doc["id"], parsed["elements"])
+    elif parsed and parsed.get("status") == "error":
+        documents.set_error(doc["id"], parsed.get("error", "parse failed"))
+        return {"ok": False, "error": f"parse failed: {parsed.get('error')}", "doc_id": doc["id"]}
+
+    if not contract_id:
+        from datetime import datetime
+        now = datetime.now()
+        c = contracts.insert(f"б/н {now.strftime('%Y%m%d')}-{now.strftime('%H%M')}")
+        contract_id = c["id"]
+        supp_type = "initial"
+    else:
+        supp_type = "additional"
+
+    supplements.insert(contract_id, doc["id"], supp_type)
+
+    return {
+        "ok": True,
+        "doc_id": doc["id"],
+        "contract_id": contract_id,
+        "filename": filename,
+        "size": len(file_data),
+        "parsed": parsed,
+        "warning": "file may be unreadable" if not parsed or not parsed.get("element_count") else None,
+    }
+
+
+# ── HTTP-handler (вызывает parse_multipart + store_document) ───
+
+def handle_upload(rfile, content_type, content_length):
+    """HTTP-handler: читает rfile → parse_multipart → store_document."""
+    try:
+        cl = int(content_length)
+    except (TypeError, ValueError):
+        return {"ok": False, "error": "invalid content-length"}
+    if cl <= 0:
+        return {"ok": False, "error": "empty request"}
+    if cl > MAX_FILE_SIZE:
+        return {"ok": False, "error": f"file too large (max {MAX_FILE_SIZE // 1024 // 1024}MB)"}
+
+    raw_bytes = rfile.read(cl)
+
+    try:
+        parts = parse_multipart(raw_bytes, content_type)
+    except ValueError as e:
+        return {"ok": False, "error": str(e)}
+
+    if not parts["filename"] or not parts["file_data"]:
+        return {"ok": False, "error": "no file in request"}
+
+    return store_document(
+        filename=parts["filename"],
+        file_data=parts["file_data"],
+        contract_id=parts["contract_id"],
+        batch_id=parts["batch_id"],
+        zip_source=parts["zip_source"],
+    )
+
+
+def _mime_for(filename):
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    mime_map = {
+        "pdf": "application/pdf",
+        "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "doc": "application/msword",
+        "zip": "application/zip",
+    }
+    return mime_map.get(ext, "application/octet-stream")