diff --git a/app/repository.py b/app/repository.py new file mode 100644 index 0000000..0977111 --- /dev/null +++ b/app/repository.py @@ -0,0 +1,268 @@ +"""Repository facade — Protocol поверх db/*.py. + +План decoupling Ф3: +- Repository (Protocol) — интерфейс доступа к данным +- PgRepository — реальная БД (обёртка над db/*.py) +- MemRepository — in-memory для юнит-тестов +- SQL не переписываем +""" +from typing import Protocol, Optional +from datetime import datetime +import uuid + + +# ── Протокол ──────────────────────────────────────────────────── + +class Repository(Protocol): + """Фасад доступа к данным.""" + + def insert_document(self, filename: str, mime_type: str, original_bytes: str, + batch_id: str = None, zip_source: str = None) -> dict: + """Вставить документ, вернуть row dict.""" + ... + + def set_document_parsed(self, doc_id: str, elements: list) -> None: + """Обновить elements_json + status='parsed'.""" + ... + + def set_document_error(self, doc_id: str, error: str) -> None: + """Обновить status='error'.""" + ... + + def set_classification(self, doc_id: str, doc_type: str, own_number: str = None, + parent_number: str = None, doc_date: str = None, + counterparty: str = None, + classify_raw: str = None, classify_input: str = None) -> None: + """Сохранить результат классификации.""" + ... + + def set_classify_garbage(self, doc_id: str, reason: str = "") -> None: + """Пометить документ как мусор.""" + ... + + def set_classify_failed(self, doc_id: str, error: str) -> None: + """Пометить классификацию как failed.""" + ... + + def list_pending(self, batch_id: str) -> list[dict]: + """Документы, ожидающие классификации.""" + ... + + def insert_contract(self, number: str, client: str = "") -> str: + """Создать контракт, вернуть contract_id.""" + ... + + def insert_supplement(self, contract_id: str, doc_id: str, supp_type: str) -> None: + """Создать связь contract↔document.""" + ... + + def list_supplements(self, contract_id: str) -> list[dict]: + """Список дополнений контракта.""" + ... + + def get_spec_current(self, contract_id: str) -> list[dict]: + """Текущая спецификация контракта.""" + ... + + def get_document(self, doc_id: str) -> dict | None: + """Получить документ по id.""" + ... + + def list_by_batch(self, batch_id: str) -> list[dict]: + """Все документы батча с полями классификации.""" + ... + + def count_by_status(self, batch_id: str) -> dict[str, int]: + """Количество документов по classify_status.""" + ... + + def reset_classify_status(self, batch_id: str) -> None: + """Сбросить classify_status на 'pending'.""" + ... + + def set_classify_processing(self, doc_id: str) -> None: + """Пометить документ как обрабатываемый.""" + ... + + def delete_document(self, doc_id: str) -> None: + """Удалить документ.""" + ... + + +# ── Продакшен: обёртка над db/*.py ────────────────────────────── + +class PgRepository: + """Реальный доступ к PostgreSQL через существующие db/*.py.""" + + def insert_document(self, filename, mime_type, original_bytes, batch_id=None, zip_source=None): + from app.db import documents + return documents.insert(filename, mime_type, original_bytes, + batch_id=batch_id, zip_source=zip_source) + + def set_document_parsed(self, doc_id, elements): + from app.db import documents + documents.set_parsed(doc_id, elements) # documents.set_parsed уже делает json.dumps + + def set_document_error(self, doc_id, error): + from app.db import documents + documents.set_error(doc_id, error) + + def set_classification(self, doc_id, doc_type, own_number=None, parent_number=None, + doc_date=None, counterparty=None, + classify_raw=None, classify_input=None): + from app.db import documents + documents.set_classification(doc_id, doc_type, own_number, parent_number, + doc_date, counterparty, + classify_raw=classify_raw, classify_input=classify_input) + + def set_classify_garbage(self, doc_id, reason=""): + from app.db import documents + documents.set_classify_garbage(doc_id, reason) + + def set_classify_failed(self, doc_id, error): + from app.db import documents + documents.set_classify_failed(doc_id, error) + + def list_pending(self, batch_id): + from app.db import documents + return documents.list_pending(batch_id) + + def insert_contract(self, number, client=""): + from app.db import contracts + c = contracts.insert(number, client) + return c["id"] if c else "" + + def insert_supplement(self, contract_id, doc_id, supp_type): + from app.db import supplements + supplements.insert(contract_id, doc_id, supp_type) + + def list_supplements(self, contract_id): + from app.db import supplements + return supplements.list_by_contract(contract_id) + + def get_spec_current(self, contract_id): + from app.db import spec_current + return spec_current.list_by_contract(contract_id) + + def get_document(self, doc_id): + from app.db import documents + return documents.get(doc_id) + + def list_by_batch(self, batch_id): + from app.db import documents + return documents.list_by_batch(batch_id) + + def count_by_status(self, batch_id): + from app.db import documents + return documents.count_by_status(batch_id) + + def reset_classify_status(self, batch_id): + from app.db import documents + documents.reset_classify_status(batch_id) + + def set_classify_processing(self, doc_id): + from app.db import documents + documents.set_classify_processing(doc_id) + + def delete_document(self, doc_id): + from app.db import documents + documents.delete(doc_id) + + +# ── Тестовый: in-memory заглушка ──────────────────────────────── + +class MemRepository: + """In-memory хранилище для юнит-тестов.""" + + def __init__(self): + self.documents: dict[str, dict] = {} + self.contracts: dict[str, dict] = {} + self.supplements: list[dict] = [] + self.spec_current: dict[str, list[dict]] = {} + + def insert_document(self, filename, mime_type, original_bytes, batch_id=None, zip_source=None): + doc_id = str(uuid.uuid4()) + self.documents[doc_id] = { + "id": doc_id, "filename": filename, "mime_type": mime_type, + "original_bytes": original_bytes, "status": "uploaded", + "elements_json": None, "doc_type": None, "own_number": None, + "parent_number": None, "doc_date": None, "counterparty": None, + "classify_status": "pending", "batch_id": batch_id, "zip_source": zip_source, + } + return self.documents[doc_id] + + def set_document_parsed(self, doc_id, elements): + if doc_id in self.documents: + self.documents[doc_id]["elements_json"] = elements + self.documents[doc_id]["status"] = "parsed" + + def set_document_error(self, doc_id, error): + if doc_id in self.documents: + self.documents[doc_id]["status"] = "error" + self.documents[doc_id]["error_message"] = error + + def set_classification(self, doc_id, doc_type, own_number=None, parent_number=None, + doc_date=None, counterparty=None, + classify_raw=None, classify_input=None): + if doc_id in self.documents: + d = self.documents[doc_id] + d.update({"doc_type": doc_type, "own_number": own_number, + "parent_number": parent_number, "doc_date": doc_date, + "counterparty": counterparty, "classify_raw": classify_raw, + "classify_input": classify_input, "classify_status": "classified"}) + + def set_classify_garbage(self, doc_id, reason=""): + if doc_id in self.documents: + self.documents[doc_id]["doc_type"] = "garbage" + self.documents[doc_id]["classify_status"] = "garbage" + + def set_classify_failed(self, doc_id, error): + if doc_id in self.documents: + self.documents[doc_id]["classify_status"] = "failed" + self.documents[doc_id]["error_message"] = error + + def list_pending(self, batch_id): + return [d for d in self.documents.values() + if d.get("batch_id") == batch_id and d.get("classify_status") == "pending"] + + def insert_contract(self, number, client=""): + cid = str(uuid.uuid4()) + self.contracts[cid] = {"id": cid, "number": number, "client": client} + return cid + + def insert_supplement(self, contract_id, doc_id, supp_type): + self.supplements.append({ + "contract_id": contract_id, "document_id": doc_id, "type": supp_type, + }) + + def list_supplements(self, contract_id): + return [s for s in self.supplements if s["contract_id"] == contract_id] + + def get_spec_current(self, contract_id): + return self.spec_current.get(contract_id, []) + + def get_document(self, doc_id): + return self.documents.get(doc_id) + + def list_by_batch(self, batch_id): + return [d for d in self.documents.values() if d.get("batch_id") == batch_id] + + def count_by_status(self, batch_id): + counts = {} + for d in self.documents.values(): + if d.get("batch_id") == batch_id: + s = d.get("classify_status", "unknown") + counts[s] = counts.get(s, 0) + 1 + return counts + + def reset_classify_status(self, batch_id): + for d in self.documents.values(): + if d.get("batch_id") == batch_id: + d["classify_status"] = "pending" + + def set_classify_processing(self, doc_id): + if doc_id in self.documents: + self.documents[doc_id]["classify_status"] = "processing" + + def delete_document(self, doc_id): + self.documents.pop(doc_id, None) diff --git a/app/services/classify.py b/app/services/classify.py new file mode 100644 index 0000000..1a435c2 --- /dev/null +++ b/app/services/classify.py @@ -0,0 +1,267 @@ +""" +Classify service — LLM-based document classification. + +Архитектурное решение (Opus): +- Отдельный сервис, не встроен в upload. Upload быстрый (0.5с), classify — медленный (2-10с/файл). +- ThreadPoolExecutor(max_workers=4) — параллельная классификация с ограничением конкурентности, + чтобы не положить api.aillm.ru при 2000 файлах. +- Умная выжимка (_smart_extract): header ~1500 симв + regex-хиты по маркерам (договор/№/соглашение) + из всего документа. Экономия токенов в 5-10 раз при сохранении точности. +- Двухпроходная архитектура: LLM извлекает строки (тип/номер/дата/контрагент), + Python в grouping.py нормализует и группирует детерминированно. +""" +import json, re, os +from concurrent.futures import ThreadPoolExecutor, as_completed + +import httpx +from app.db import documents as db_docs +from app.llm_prompt import build_classify_prompt + +log = __import__("logging").getLogger(__name__) + +# Лимит одновременных запросов к LLM API +# Увеличивать осторожно — api.aillm.ru может троттлить +MAX_WORKERS = 4 + +LLM_URL = "https://api.aillm.ru/v1/chat/completions" +LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "") +LLM_MODEL = "gpt-oss-120b" + +# Ленивый singleton — обратная совместимость +_classify_llm = None + + +def _get_classify_client(): + global _classify_llm + if _classify_llm is None: + from app.services.llm_client import HttpxLLMClient + _classify_llm = HttpxLLMClient(url=LLM_URL, key=LLM_KEY, model=LLM_MODEL, max_tokens=1000, timeout=60) + return _classify_llm + +# ── Garbage filter (Stage 1: filename regex) ──────────────────── +_GARBAGE_FILENAME_RE = re.compile( + r'(сч[её]т|акт|плат[её]ж|УПД|сверк|инвойс|invoice|payment|act)', + re.IGNORECASE, +) + +# ── Garbage filter (Stage 2: header keywords) ─────────────────── +_GARBAGE_HEADER_MARKERS = [ + 'СЧЕТ-ФАКТУРА', 'СЧЕТ НА ОПЛАТУ', 'АКТ СВЕРКИ', + 'АКТ ОКАЗАННЫХ УСЛУГ', 'АКТ ВЫПОЛНЕННЫХ РАБОТ', + 'ПЛАТЁЖНОЕ ПОРУЧЕНИЕ', 'УНИВЕРСАЛЬНЫЙ ПЕРЕДАТОЧНЫЙ', + 'УПД', 'ПЛАТЕЖНОЕ ПОРУЧЕНИЕ', +] + + +def _is_garbage_by_filename(filename: str) -> bool: + """Stage 1: regex по имени файла — быстро, 0 токенов.""" + return bool(_GARBAGE_FILENAME_RE.search(filename)) + + +def _is_garbage_by_header(text: str) -> bool: + """Stage 2: ключевые слова в первых 2KB текста — быстро, 0 токенов.""" + header = text[:2000].upper() + return any(marker in header for marker in _GARBAGE_HEADER_MARKERS) + + +def _call_llm_classify(header_text, llm_client=None): + """ + Прямой вызов LLM для классификации ОДНОГО документа. + Возвращает (parsed_dict, raw_text, needed_fix). + llm_client: LLMClient (optional). Default — HttpxLLMClient. + """ + if llm_client is None: + llm_client = _get_classify_client() + + prompt, _ = build_classify_prompt(header_text) + raw_text = llm_client.complete(prompt) + parsed, needed_fix = _safe_json_parse(raw_text) + return parsed, raw_text, needed_fix + + +def classify_batch(batch_id, llm_client=None, repo=None): + """ + Классифицировать все документы в batch. + Сбрасывает статус на 'pending' для всех перед началом. + Параллельно (ThreadPoolExecutor) обрабатывает до MAX_WORKERS документов. + Возвращает {ok, total, done, failed, garbage, json_fix_rate}. + llm_client: LLMClient (optional, default — HttpxLLMClient) + repo: Repository (optional, default — direct db.* calls) + """ + _db = repo if repo else db_docs + _llm = llm_client if llm_client else _get_classify_client() + + # Сбросить статус — allow re-classify after file changes + _db.reset_classify_status(batch_id) + pending = _db.list_pending(batch_id) + if not pending: + return {"ok": False, "error": "no pending documents"} + + total = len(pending) + done = 0 + failed = 0 + garbage = 0 + json_fixes = 0 + json_total = 0 + type_counts = {} # doc_type → count for batch summary + + def _classify_one(doc): + """Классифицировать один документ: фильтр → выжимка → LLM → сохранить.""" + nonlocal garbage, json_fixes, json_total, type_counts + try: + # Stage 1: garbage by filename (0 tokens) + if _is_garbage_by_filename(doc["filename"]): + _db.set_classify_garbage(doc["id"], "filename_regex") + garbage += 1 + return True + + # Stage 2: garbage by header keywords (0 tokens) + text = _smart_extract(doc["elements_json"]) + if _is_garbage_by_header(text): + _db.set_classify_garbage(doc["id"], "header_keywords") + garbage += 1 + return True + + # Stage 3: LLM classification (only for remaining) + _db.set_classify_processing(doc["id"]) # crash recovery marker + result, raw, needed_fix = _call_llm_classify(text, _llm) + json_total += 1 + if needed_fix: + json_fixes += 1 + dtype = result.get("doc_type", "other") + type_counts[dtype] = type_counts.get(dtype, 0) + 1 + _db.set_classification( + doc["id"], + result.get("doc_type", "other"), + result.get("own_number"), + result.get("parent_number"), + result.get("doc_date"), + result.get("counterparty"), + classify_raw=raw, + classify_input=text, + ) + return True + except Exception as e: + _db.set_classify_failed(doc["id"], str(e)) + return False + + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool: + futures = {pool.submit(_classify_one, d): d for d in pending} + for f in as_completed(futures): + if f.result(): + done += 1 + else: + failed += 1 + + return {"ok": True, "total": total, "done": done, "failed": failed, + "garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3), + "types": type_counts, "summary": f"{total} total, {done} classified, {failed} failed, {garbage} garbage"} + + +def _safe_json_parse(raw): + """Parse LLM response, fixing common JSON errors. + Returns (parsed_dict, needed_fix: bool).""" + if not raw: + raise ValueError("empty LLM response") + + text = raw.strip() + # Strip markdown + if "```json" in text: + text = text.split("```json")[1].split("```")[0].strip() + elif "```" in text: + text = text.split("```")[1].split("```")[0].strip() + + # Remove non-JSON prefix/suffix (LLM chatter) + brace_start = text.find("{") + brace_end = text.rfind("}") + if brace_start >= 0 and brace_end > brace_start: + text = text[brace_start:brace_end + 1] + + # Try strict parse + try: + return json.loads(text), False + except json.JSONDecodeError: + pass + + import re as _re + # Collapse multiline + text = _re.sub(r"\n\s*", " ", text) + # Remove trailing commas + text = _re.sub(r",\s*}", "}", text) + text = _re.sub(r",\s*]", "]", text) + + # Try again + try: + return json.loads(text), True + except json.JSONDecodeError: + pass + + # Aggressive: try adding missing closing quotes/braces + text = text.rstrip() + if not text.endswith("}"): + # Count unclosed quotes + in_string = False + for i, ch in enumerate(text): + if ch == '"' and (i == 0 or text[i-1] != "\\"): + in_string = not in_string + if in_string: + text += '"' + text += "}" + + return json.loads(text), True + + +def _smart_extract(elements_json): + """ + Умная выжимка текста для классификации (решение Q3 от Opus). + + Вместо отправки всего документа (дорого) или только header (теряет зарытые номера), + используется гибрид: + 1. Первые ~1500 симв (титул, преамбула, стороны) + 2. Regex-хиты по маркерам «договор|№|соглашение|приложение|спецификация» + из ВСЕГО документа + 3. Дедупликация, лимит 10 строк, склейка → ~3000 симв на вход LLM + + Это покрывает и титульную зону, и зарытые ссылки в середине документа. + """ + if not elements_json: + return "" + + if isinstance(elements_json, str): + try: + elements = json.loads(elements_json) + except json.JSONDecodeError: + return elements_json[:2000] + elif isinstance(elements_json, list): + elements = elements_json + else: + return str(elements_json)[:2000] + + # Build full text + lines = [] + for el in elements: + if isinstance(el, dict): + t = el.get("type") or el.get("TYPE", "") + if t == "paragraph": + txt = el.get("text") or el.get("TEXT", "") + if txt: + lines.append(txt) + elif t == "table": + rows = el.get("rows") or el.get("ROWS", []) + for row in rows: + lines.append(" | ".join(str(c) for c in row)) + + full_text = "\n".join(lines) + + # Header: first ~1500 chars + header = full_text[:1500] + + # Marker lines: grep for key patterns + markers = re.findall( + r'.{0,200}(?:договор|№|соглашен|приложен|специф|контрагент|заказчик|арендатор).{0,200}', + full_text, re.IGNORECASE, + ) + unique_markers = list(dict.fromkeys(markers))[:10] + + combined = header + "\n---\n" + "\n".join(unique_markers) + return combined[:3000] diff --git a/app/services/grouping.py b/app/services/grouping.py new file mode 100644 index 0000000..c7fe36c --- /dev/null +++ b/app/services/grouping.py @@ -0,0 +1,160 @@ +""" +Grouping service — match classified documents into contract groups. + +Архитектурное решение (Opus, Q4 + Q6): +- Двухпроходный гибрид: LLM извлекает строки (classify.py), Python нормализует и группирует. +- Нормализация номеров: uppercase + только буквы/цифры. + "МЭС-123-2024" == "МЭС 123/2024" после нормализации. +- Группировка на бэкенде (не на фронте): Python regex/unicode надёжнее JS. +- apply_groups(): создаёт contracts + supplements с авто-порядком по дате. +""" +import re +from app.db import documents as db_docs +from app.db import contracts as db_contracts +from app.db import supplements as db_supplements + + +def normalize_number(num): + """ + Нормализация номера договора для сравнения. + Убирает всё кроме букв и цифр, приводит к uppercase. + Пример: "МЭС-123-2024" → "МЭС1232024", "МЭС 123/2024" → "МЭС1232024". + """ + if not num: + return "" + return re.sub(r"[^A-Z0-9А-Я]", "", num.upper()) + + +def group_documents(batch_id): + """ + Сгруппировать классифицированные документы по контрактам. + + Алгоритм: + 1. Отделить contract от supplement/specification + 2. Каждый contract → якорь группы + 3. Для каждого supplement: найти contract по parent_number (нормализованный) + 4. Оставшиеся supplement/spec → виртуальные группы по parent_number/own_number + 5. Совсем без номеров → группа "__unresolved__" + 6. Внутри группы сортировка по doc_date + + Возвращает {ok, groups: [{contract_number, counterparty, documents: [...]}], total_docs}. + """ + docs = db_docs.list_by_batch(batch_id) + classified = [d for d in docs if d.get("classify_status") == "classified"] + + # Separate contracts and supplements + contracts_list = [] + supplements_list = [] + + for d in classified: + if d.get("doc_type") == "contract": + contracts_list.append(d) + else: + supplements_list.append(d) + + groups = [] + + # Each contract becomes a group + for c in contracts_list: + group = { + "contract_number": c.get("own_number") or c.get("filename", ""), + "counterparty": c.get("counterparty") or "", + "documents": [c], + } + # Find supplements matching this contract + c_norm = normalize_number(c.get("own_number")) + for s in supplements_list: + parent = normalize_number(s.get("parent_number") or "") + own = normalize_number(s.get("own_number") or "") + if parent == c_norm or own == c_norm: + if s not in group["documents"]: + group["documents"].append(s) + + # Sort by date + group["documents"].sort(key=lambda x: x.get("doc_date") or "") + + # Remove matched supplements from the pool + for s in group["documents"]: + if s in supplements_list: + supplements_list.remove(s) + + groups.append(group) + + # ── Virtual groups: unmatched supplements grouped by number ────────── + # Group remaining supplements/specs by normalized parent_number (priority) or own_number + virtual = {} + for s in supplements_list: + num = normalize_number(s.get("parent_number") or s.get("own_number") or "") + if not num: + continue # no number → stays in supplements_list for unresolved + if num not in virtual: + # Use counterparty from first doc in group + cp = s.get("counterparty") or "" + virtual[num] = { + "contract_number": s.get("parent_number") or s.get("own_number") or "?", + "counterparty": cp, + "documents": [], + } + virtual[num]["documents"].append(s) + # Update counterparty if current doc has a better one + if not virtual[num]["counterparty"] and s.get("counterparty"): + virtual[num]["counterparty"] = s.get("counterparty") + + for vnum, vgroup in virtual.items(): + vgroup["documents"].sort(key=lambda x: x.get("doc_date") or "") + groups.append(vgroup) + # Remove grouped docs from supplements_list + for s in vgroup["documents"]: + supplements_list.remove(s) + + # ── Unresolved: everything left (no number, failed, pending, other) ── + unmatched = [s for s in supplements_list] # remaining after virtual grouping + unmatched += [d for d in docs if d.get("classify_status") != "classified"] + + if unmatched: + groups.append({ + "contract_number": "__unresolved__", + "counterparty": "", + "documents": unmatched, + }) + + return {"ok": True, "groups": groups, "total_docs": len(docs)} + + +def apply_groups(batch_id, groups_data): + """ + Применить подтверждённые группы: создать contracts + supplements. + + Вызывается из POST /api/apply-groups. + Для каждой группы (кроме __unresolved__): + 1. Создать запись в contracts (number, client) + 2. Для каждого документа создать supplement (type='initial' для первого, 'additional' для остальных) + 3. Порядок supplements соответствует порядку документов в группе (сортировка по дате уже сделана) + + Возвращает {ok, created: количество созданных supplements}. + """ + created = 0 + contract_ids = [] + for g in groups_data: + contract_number = g.get("contract_number", "") + if contract_number == "__unresolved__": + continue + counterparty = g.get("counterparty", "") + docs = g.get("documents", []) + + try: + c = db_contracts.insert(contract_number, counterparty) + contract_id = c["id"] + contract_ids.append(contract_id) + + for i, d in enumerate(docs): + doc_id = d.get("id") + if not doc_id: + continue + supp_type = "initial" if i == 0 else "additional" + db_supplements.insert(contract_id, doc_id, supp_type) + created += 1 + except Exception as e: + return {"ok": False, "error": f"apply_groups failed at {contract_number}: {e}"} + + return {"ok": True, "created": created, "contract_ids": contract_ids} diff --git a/app/services/llm.py b/app/services/llm.py new file mode 100644 index 0000000..4fc4226 --- /dev/null +++ b/app/services/llm.py @@ -0,0 +1,35 @@ +"""LLM service — call LLM API with optional DI.""" +import os, json + +LLM_URL = "https://api.aillm.ru/v1/chat/completions" +LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "") +LLM_MODEL = "gpt-oss-120b" + +# Ленивый singleton — обратная совместимость +_llm_client = None + + +def _get_default_client(): + global _llm_client + if _llm_client is None: + from app.services.llm_client import HttpxLLMClient + _llm_client = HttpxLLMClient(url=LLM_URL, key=LLM_KEY, model=LLM_MODEL) + return _llm_client + + +def call_llm(current_spec, doc_text, build_prompt_fn, llm_client=None): + """Call LLM. Returns (parsed_result, prompt_id). + llm_client: LLMClient (optional). Default — HttpxLLMClient (prod). + """ + if llm_client is None: + llm_client = _get_default_client() + + prompt, prompt_id = build_prompt_fn(current_spec, doc_text) + raw_text = llm_client.complete(prompt) + + json_text = raw_text + if "```json" in json_text: + json_text = json_text.split("```json")[1].split("```")[0] + elif "```" in json_text: + json_text = json_text.split("```")[1].split("```")[0] + return json.loads(json_text.strip()), prompt_id diff --git a/app/services/metrics.py b/app/services/metrics.py new file mode 100644 index 0000000..4f389fa --- /dev/null +++ b/app/services/metrics.py @@ -0,0 +1,88 @@ +"""Metrics — quality signals without golden dataset. + +Checks that work immediately: +- Arithmetic: sum == price * qty (free signal of LLM/data errors) +- JSON fix rate: how often _safe_json_parse has to repair LLM output +""" +from decimal import Decimal, InvalidOperation + + +def check_arithmetic(ops: list) -> list[dict]: + """Check sum == price * qty for ADD/UPDATE operations. + Returns list of mismatches: [{action, name, price, qty, expected_sum, actual_sum, diff}] + """ + mismatches = [] + for op in ops: + action = op.get("action", "") + if action not in ("ADD", "UPDATE"): + continue + + row = op.get("new_row") or op.get("new_values") or {} + price = _to_decimal(row.get("price")) + qty = _to_decimal(row.get("qty")) + actual_sum = _to_decimal(row.get("sum")) + + if price is None or qty is None or actual_sum is None: + continue # can't check without all three + + expected = price * qty + if expected != actual_sum: + mismatches.append({ + "action": action, + "name": row.get("name", "")[:100], + "price": float(price), + "qty": float(qty), + "expected_sum": float(expected), + "actual_sum": float(actual_sum), + "diff": float(actual_sum - expected), + }) + return mismatches + + +class ClassifyMetrics: + """Track _safe_json_parse fix rate per batch.""" + def __init__(self): + self.total = 0 + self.fixes = 0 # how many times JSON needed repair + + def record(self, needed_fix: bool): + self.total += 1 + if needed_fix: + self.fixes += 1 + + @property + def fix_rate(self) -> float: + return self.fixes / self.total if self.total else 0.0 + + def summary(self) -> dict: + return { + "total_classifications": self.total, + "json_fixes": self.fixes, + "json_fix_rate": round(self.fix_rate, 3), + } + + +def _to_decimal(val) -> Decimal | None: + """Safe conversion to Decimal with number normalization.""" + if val is None or val == "": + return None + try: + s = str(val).replace("\xa0", "").replace(" ", "").replace(",", ".") + return Decimal(s) + except (InvalidOperation, ValueError): + return None + + +def normalize_date(ds: str) -> str | None: + """Normalize date to YYYY-MM-DD. Handles DD.MM.YYYY, YYYY-MM-DD, etc.""" + if not ds: + return None + import re + # DD.MM.YYYY → YYYY-MM-DD + m = re.match(r"(\d{2})\.(\d{2})\.(\d{4})", ds) + if m: + return f"{m.group(3)}-{m.group(2)}-{m.group(1)}" + # YYYY-MM-DD — already canonical + if re.match(r"\d{4}-\d{2}-\d{2}", ds): + return ds + return ds # return as-is if unrecognized format diff --git a/app/services/parse.py b/app/services/parse.py new file mode 100644 index 0000000..20cc37e --- /dev/null +++ b/app/services/parse.py @@ -0,0 +1,87 @@ +"""Parse service — PDF (pdfplumber), DOCX (python-docx), plain text fallback.""" +import io + + +def parse_file(filename, data): + """Parse file bytes → {status, elements, element_count}.""" + ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + try: + if ext == "pdf": + return _parse_pdf(data) + elif ext == "docx": + return _parse_docx(data) + elif ext == "doc": + return _parse_docx(data) # python-docx handles most .doc + else: + return _parse_text(data) + except Exception as e: + return {"status": "error", "error": str(e), "element_count": 0} + + +def _parse_pdf(data): + """Parse PDF with pdfplumber — preserves table structure (unlike PyPDF2).""" + import pdfplumber + elements = [] + with pdfplumber.open(io.BytesIO(data)) as pdf: + for page in pdf.pages: + # Tables first — preserves column structure critical for specs + tables = page.extract_tables() + for table in tables: + if table: + rows = [] + for row in table: + if row: + cells = [str(cell or "").strip() for cell in row] + if any(cells): + rows.append(cells) + if rows: + elements.append({"type": "table", "rows": rows}) + # Remaining text as paragraphs + text = page.extract_text() + if text: + for line in text.split("\n"): + line = line.strip() + if line: + elements.append({"type": "paragraph", "text": line, "style": ""}) + return {"status": "parsed", "element_count": len(elements), "elements": elements} + + +def _parse_docx(data): + import docx + doc = docx.Document(io.BytesIO(data)) + elements = [] + + for block in doc.element.body: + tag = block.tag.split("}")[-1] if "}" in block.tag else block.tag + if tag == "p": + text = _extract_paragraph_text(block) + if text: + elements.append({"type": "paragraph", "text": text, "style": ""}) + elif tag == "tbl": + rows = [] + for tr in block.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tr"): + cells = [] + for tc in tr.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tc"): + cell_text = "".join(t.text or "" for t in tc.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t")) + cells.append(cell_text.strip()) + if cells: + rows.append(cells) + if rows: + elements.append({"type": "table", "rows": rows}) + + return {"status": "parsed", "element_count": len(elements), "elements": elements} + + +def _extract_paragraph_text(p_elem): + texts = [] + for t in p_elem.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t"): + if t.text: + texts.append(t.text) + return "".join(texts).strip() + + +def _parse_text(data): + text = data.decode("utf-8", errors="ignore")[:5000] + lines = [l.strip() for l in text.split("\n") if l.strip()] + return {"status": "parsed", "element_count": len(lines), + "elements": [{"type": "paragraph", "text": l, "style": ""} for l in lines]}