fix(ревью#2): db_docs→_db + classify_raw в MemRepo + try/except в build_classify_prompt
This commit is contained in:
@@ -226,12 +226,14 @@ def build_prompt(current_spec: list, doc_text: str) -> tuple:
|
||||
|
||||
def build_classify_prompt(header_text):
|
||||
"""Build classify prompt. Returns (prompt, prompt_id)."""
|
||||
from db import prompts as db_prompts
|
||||
prompt = db_prompts.get_active("classify")
|
||||
if prompt:
|
||||
body = prompt["body"].replace("{header_text}", header_text)
|
||||
return body, prompt.get("id", "")
|
||||
# Fallback
|
||||
try:
|
||||
from db import prompts as db_prompts
|
||||
prompt = db_prompts.get_active("classify")
|
||||
if prompt:
|
||||
body = prompt["body"].replace("{header_text}", header_text)
|
||||
return body, prompt.get("id", "")
|
||||
except Exception:
|
||||
pass # DB unavailable — use fallback
|
||||
body = """Ты — классификатор договорных документов облачного провайдера НУБЕС.
|
||||
|
||||
Ниже фрагмент текста документа. Определи:
|
||||
|
||||
+12
-10
@@ -31,7 +31,8 @@ class Repository(Protocol):
|
||||
|
||||
def set_classification(self, doc_id: str, doc_type: str, own_number: str = None,
|
||||
parent_number: str = None, doc_date: str = None,
|
||||
counterparty: str = None) -> None:
|
||||
counterparty: str = None,
|
||||
classify_raw: str = None, classify_input: str = None) -> None:
|
||||
"""Сохранить результат классификации."""
|
||||
...
|
||||
|
||||
@@ -107,10 +108,12 @@ class PgRepository:
|
||||
documents.set_error(doc_id, error)
|
||||
|
||||
def set_classification(self, doc_id, doc_type, own_number=None, parent_number=None,
|
||||
doc_date=None, counterparty=None):
|
||||
doc_date=None, counterparty=None,
|
||||
classify_raw=None, classify_input=None):
|
||||
from db import documents
|
||||
documents.set_classification(doc_id, doc_type, own_number, parent_number,
|
||||
doc_date, counterparty)
|
||||
doc_date, counterparty,
|
||||
classify_raw=classify_raw, classify_input=classify_input)
|
||||
|
||||
def set_classify_garbage(self, doc_id, reason=""):
|
||||
from db import documents
|
||||
@@ -199,15 +202,14 @@ class MemRepository:
|
||||
self.documents[doc_id]["error_message"] = error
|
||||
|
||||
def set_classification(self, doc_id, doc_type, own_number=None, parent_number=None,
|
||||
doc_date=None, counterparty=None):
|
||||
doc_date=None, counterparty=None,
|
||||
classify_raw=None, classify_input=None):
|
||||
if doc_id in self.documents:
|
||||
d = self.documents[doc_id]
|
||||
d["doc_type"] = doc_type
|
||||
d["own_number"] = own_number
|
||||
d["parent_number"] = parent_number
|
||||
d["doc_date"] = doc_date
|
||||
d["counterparty"] = counterparty
|
||||
d["classify_status"] = "classified"
|
||||
d.update({"doc_type": doc_type, "own_number": own_number,
|
||||
"parent_number": parent_number, "doc_date": doc_date,
|
||||
"counterparty": counterparty, "classify_raw": classify_raw,
|
||||
"classify_input": classify_input, "classify_status": "classified"})
|
||||
|
||||
def set_classify_garbage(self, doc_id, reason=""):
|
||||
if doc_id in self.documents:
|
||||
|
||||
@@ -92,8 +92,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
_llm = llm_client if llm_client else _get_classify_client()
|
||||
|
||||
# Сбросить статус — allow re-classify after file changes
|
||||
db_docs.reset_classify_status(batch_id)
|
||||
pending = db_docs.list_pending(batch_id)
|
||||
_db.reset_classify_status(batch_id)
|
||||
pending = _db.list_pending(batch_id)
|
||||
if not pending:
|
||||
return {"ok": False, "error": "no pending documents"}
|
||||
|
||||
@@ -110,24 +110,24 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
try:
|
||||
# Stage 1: garbage by filename (0 tokens)
|
||||
if _is_garbage_by_filename(doc["filename"]):
|
||||
db_docs.set_classify_garbage(doc["id"], "filename_regex")
|
||||
_db.set_classify_garbage(doc["id"], "filename_regex")
|
||||
garbage += 1
|
||||
return True
|
||||
|
||||
# Stage 2: garbage by header keywords (0 tokens)
|
||||
text = _smart_extract(doc["elements_json"])
|
||||
if _is_garbage_by_header(text):
|
||||
db_docs.set_classify_garbage(doc["id"], "header_keywords")
|
||||
_db.set_classify_garbage(doc["id"], "header_keywords")
|
||||
garbage += 1
|
||||
return True
|
||||
|
||||
# Stage 3: LLM classification (only for remaining)
|
||||
db_docs.set_classify_processing(doc["id"]) # crash recovery marker
|
||||
_db.set_classify_processing(doc["id"]) # crash recovery marker
|
||||
result, raw, needed_fix = _call_llm_classify(text, _llm)
|
||||
json_total += 1
|
||||
if needed_fix:
|
||||
json_fixes += 1
|
||||
db_docs.set_classification(
|
||||
_db.set_classification(
|
||||
doc["id"],
|
||||
result.get("doc_type", "other"),
|
||||
result.get("own_number"),
|
||||
@@ -139,7 +139,7 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
db_docs.set_classify_failed(doc["id"], str(e))
|
||||
_db.set_classify_failed(doc["id"], str(e))
|
||||
return False
|
||||
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Test classify_batch with MemRepository + FakeLLM."""
|
||||
from services.classify import classify_batch
|
||||
from services.llm_client import FakeLLMClient
|
||||
from repository import MemRepository
|
||||
|
||||
|
||||
class TestClassifyBatch:
|
||||
def test_garbage_by_filename(self):
|
||||
"""Счёт-фактура отфильтровывается без LLM."""
|
||||
repo = MemRepository()
|
||||
llm = FakeLLMClient({"default": '{"doc_type":"contract"}'})
|
||||
|
||||
repo.insert_document("счет-фактура №123.docx", "mime", "data", batch_id="b1")
|
||||
|
||||
result = classify_batch("b1", llm_client=llm, repo=repo)
|
||||
assert result["ok"] is True
|
||||
assert result["garbage"] == 1
|
||||
assert result["done"] == 1
|
||||
assert len(llm.calls) == 0 # LLM не вызывался
|
||||
|
||||
def test_garbage_by_header(self):
|
||||
"""Акт сверки в тексте отфильтровывается."""
|
||||
repo = MemRepository()
|
||||
llm = FakeLLMClient({"default": '{"doc_type":"contract"}'})
|
||||
|
||||
doc = repo.insert_document("документ.docx", "mime", "data", batch_id="b1")
|
||||
repo.set_document_parsed(doc["id"], [
|
||||
{"type": "paragraph", "text": "АКТ СВЕРКИ взаимных расчётов", "style": ""}
|
||||
])
|
||||
|
||||
result = classify_batch("b1", llm_client=llm, repo=repo)
|
||||
assert result["garbage"] == 1
|
||||
assert len(llm.calls) == 0
|
||||
|
||||
def test_llm_classify(self):
|
||||
"""Договор классифицируется через LLM."""
|
||||
repo = MemRepository()
|
||||
llm = FakeLLMClient({
|
||||
"default": '{"doc_type":"contract","own_number":"03700_1","doc_date":"2026-02-01","counterparty":"ЗАО XXX001"}'
|
||||
})
|
||||
|
||||
doc = repo.insert_document("договор-XXX001.docx", "mime", "data", batch_id="b1")
|
||||
repo.set_document_parsed(doc["id"], [
|
||||
{"type": "paragraph", "text": "ДОГОВОР № 03700_1", "style": ""}
|
||||
])
|
||||
|
||||
result = classify_batch("b1", llm_client=llm, repo=repo)
|
||||
assert result["done"] == 1, f"done={result['done']}, failed={result['failed']}, doc={repo.get_document(doc['id'])}"
|
||||
assert result["garbage"] == 0
|
||||
assert len(llm.calls) == 1
|
||||
|
||||
updated = repo.get_document(doc["id"])
|
||||
assert updated["doc_type"] == "contract"
|
||||
assert updated["own_number"] == "03700_1"
|
||||
assert updated["counterparty"] == "ЗАО XXX001"
|
||||
|
||||
def test_no_pending(self):
|
||||
"""Пустой батч."""
|
||||
repo = MemRepository()
|
||||
result = classify_batch("empty", llm_client=FakeLLMClient(), repo=repo)
|
||||
assert result["ok"] is False
|
||||
assert "no pending" in result["error"]
|
||||
Reference in New Issue
Block a user