From d8e8b35092fad6f37ed36d95ddbf80849b20f917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Sun, 28 Jun 2026 09:40:00 +0400 Subject: [PATCH] =?UTF-8?q?fix(=D1=80=D0=B5=D0=B2=D1=8C=D1=8E#2):=20db=5Fd?= =?UTF-8?q?ocs=E2=86=92=5Fdb=20+=20classify=5Fraw=20=D0=B2=20MemRepo=20+?= =?UTF-8?q?=20try/except=20=D0=B2=20build=5Fclassify=5Fprompt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/llm_prompt.py | 14 +++--- deploy/repository.py | 22 +++++---- deploy/services/classify.py | 14 +++--- deploy/tests/unit/test_classify_batch.py | 62 ++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 23 deletions(-) create mode 100644 deploy/tests/unit/test_classify_batch.py diff --git a/deploy/llm_prompt.py b/deploy/llm_prompt.py index 5731efd..7b64351 100644 --- a/deploy/llm_prompt.py +++ b/deploy/llm_prompt.py @@ -226,12 +226,14 @@ def build_prompt(current_spec: list, doc_text: str) -> tuple: def build_classify_prompt(header_text): """Build classify prompt. Returns (prompt, prompt_id).""" - from db import prompts as db_prompts - prompt = db_prompts.get_active("classify") - if prompt: - body = prompt["body"].replace("{header_text}", header_text) - return body, prompt.get("id", "") - # Fallback + try: + from db import prompts as db_prompts + prompt = db_prompts.get_active("classify") + if prompt: + body = prompt["body"].replace("{header_text}", header_text) + return body, prompt.get("id", "") + except Exception: + pass # DB unavailable — use fallback body = """Ты — классификатор договорных документов облачного провайдера НУБЕС. Ниже фрагмент текста документа. Определи: diff --git a/deploy/repository.py b/deploy/repository.py index c662271..9f3f6e5 100644 --- a/deploy/repository.py +++ b/deploy/repository.py @@ -31,7 +31,8 @@ class Repository(Protocol): def set_classification(self, doc_id: str, doc_type: str, own_number: str = None, parent_number: str = None, doc_date: str = None, - counterparty: str = None) -> None: + counterparty: str = None, + classify_raw: str = None, classify_input: str = None) -> None: """Сохранить результат классификации.""" ... @@ -107,10 +108,12 @@ class PgRepository: documents.set_error(doc_id, error) def set_classification(self, doc_id, doc_type, own_number=None, parent_number=None, - doc_date=None, counterparty=None): + doc_date=None, counterparty=None, + classify_raw=None, classify_input=None): from db import documents documents.set_classification(doc_id, doc_type, own_number, parent_number, - doc_date, counterparty) + doc_date, counterparty, + classify_raw=classify_raw, classify_input=classify_input) def set_classify_garbage(self, doc_id, reason=""): from db import documents @@ -199,15 +202,14 @@ class MemRepository: self.documents[doc_id]["error_message"] = error def set_classification(self, doc_id, doc_type, own_number=None, parent_number=None, - doc_date=None, counterparty=None): + doc_date=None, counterparty=None, + classify_raw=None, classify_input=None): if doc_id in self.documents: d = self.documents[doc_id] - d["doc_type"] = doc_type - d["own_number"] = own_number - d["parent_number"] = parent_number - d["doc_date"] = doc_date - d["counterparty"] = counterparty - d["classify_status"] = "classified" + d.update({"doc_type": doc_type, "own_number": own_number, + "parent_number": parent_number, "doc_date": doc_date, + "counterparty": counterparty, "classify_raw": classify_raw, + "classify_input": classify_input, "classify_status": "classified"}) def set_classify_garbage(self, doc_id, reason=""): if doc_id in self.documents: diff --git a/deploy/services/classify.py b/deploy/services/classify.py index 1edb347..ab70d56 100644 --- a/deploy/services/classify.py +++ b/deploy/services/classify.py @@ -92,8 +92,8 @@ def classify_batch(batch_id, llm_client=None, repo=None): _llm = llm_client if llm_client else _get_classify_client() # Сбросить статус — allow re-classify after file changes - db_docs.reset_classify_status(batch_id) - pending = db_docs.list_pending(batch_id) + _db.reset_classify_status(batch_id) + pending = _db.list_pending(batch_id) if not pending: return {"ok": False, "error": "no pending documents"} @@ -110,24 +110,24 @@ def classify_batch(batch_id, llm_client=None, repo=None): try: # Stage 1: garbage by filename (0 tokens) if _is_garbage_by_filename(doc["filename"]): - db_docs.set_classify_garbage(doc["id"], "filename_regex") + _db.set_classify_garbage(doc["id"], "filename_regex") garbage += 1 return True # Stage 2: garbage by header keywords (0 tokens) text = _smart_extract(doc["elements_json"]) if _is_garbage_by_header(text): - db_docs.set_classify_garbage(doc["id"], "header_keywords") + _db.set_classify_garbage(doc["id"], "header_keywords") garbage += 1 return True # Stage 3: LLM classification (only for remaining) - db_docs.set_classify_processing(doc["id"]) # crash recovery marker + _db.set_classify_processing(doc["id"]) # crash recovery marker result, raw, needed_fix = _call_llm_classify(text, _llm) json_total += 1 if needed_fix: json_fixes += 1 - db_docs.set_classification( + _db.set_classification( doc["id"], result.get("doc_type", "other"), result.get("own_number"), @@ -139,7 +139,7 @@ def classify_batch(batch_id, llm_client=None, repo=None): ) return True except Exception as e: - db_docs.set_classify_failed(doc["id"], str(e)) + _db.set_classify_failed(doc["id"], str(e)) return False with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool: diff --git a/deploy/tests/unit/test_classify_batch.py b/deploy/tests/unit/test_classify_batch.py new file mode 100644 index 0000000..f710c58 --- /dev/null +++ b/deploy/tests/unit/test_classify_batch.py @@ -0,0 +1,62 @@ +"""Test classify_batch with MemRepository + FakeLLM.""" +from services.classify import classify_batch +from services.llm_client import FakeLLMClient +from repository import MemRepository + + +class TestClassifyBatch: + def test_garbage_by_filename(self): + """Счёт-фактура отфильтровывается без LLM.""" + repo = MemRepository() + llm = FakeLLMClient({"default": '{"doc_type":"contract"}'}) + + repo.insert_document("счет-фактура №123.docx", "mime", "data", batch_id="b1") + + result = classify_batch("b1", llm_client=llm, repo=repo) + assert result["ok"] is True + assert result["garbage"] == 1 + assert result["done"] == 1 + assert len(llm.calls) == 0 # LLM не вызывался + + def test_garbage_by_header(self): + """Акт сверки в тексте отфильтровывается.""" + repo = MemRepository() + llm = FakeLLMClient({"default": '{"doc_type":"contract"}'}) + + doc = repo.insert_document("документ.docx", "mime", "data", batch_id="b1") + repo.set_document_parsed(doc["id"], [ + {"type": "paragraph", "text": "АКТ СВЕРКИ взаимных расчётов", "style": ""} + ]) + + result = classify_batch("b1", llm_client=llm, repo=repo) + assert result["garbage"] == 1 + assert len(llm.calls) == 0 + + def test_llm_classify(self): + """Договор классифицируется через LLM.""" + repo = MemRepository() + llm = FakeLLMClient({ + "default": '{"doc_type":"contract","own_number":"03700_1","doc_date":"2026-02-01","counterparty":"ЗАО XXX001"}' + }) + + doc = repo.insert_document("договор-XXX001.docx", "mime", "data", batch_id="b1") + repo.set_document_parsed(doc["id"], [ + {"type": "paragraph", "text": "ДОГОВОР № 03700_1", "style": ""} + ]) + + result = classify_batch("b1", llm_client=llm, repo=repo) + assert result["done"] == 1, f"done={result['done']}, failed={result['failed']}, doc={repo.get_document(doc['id'])}" + assert result["garbage"] == 0 + assert len(llm.calls) == 1 + + updated = repo.get_document(doc["id"]) + assert updated["doc_type"] == "contract" + assert updated["own_number"] == "03700_1" + assert updated["counterparty"] == "ЗАО XXX001" + + def test_no_pending(self): + """Пустой батч.""" + repo = MemRepository() + result = classify_batch("empty", llm_client=FakeLLMClient(), repo=repo) + assert result["ok"] is False + assert "no pending" in result["error"]