From 31e5ef7ac5834957e44af0cf149d836dedf841ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Sat, 27 Jun 2026 13:18:10 +0400 Subject: [PATCH] =?UTF-8?q?feat:=20=D0=BC=D0=B5=D1=82=D1=80=D0=B8=D0=BA?= =?UTF-8?q?=D0=B8=20=D0=BA=D0=B0=D1=87=D0=B5=D1=81=D1=82=D0=B2=D0=B0=20?= =?UTF-8?q?=E2=80=94=20=D0=B0=D1=80=D0=B8=D1=84=D0=BC=D0=B5=D1=82=D0=B8?= =?UTF-8?q?=D0=BA=D0=B0=20(sum=3D=3Dprice*qty)=20+=20=D1=87=D0=B0=D1=81?= =?UTF-8?q?=D1=82=D0=BE=D1=82=D0=B0=20JSON-=D1=84=D0=B8=D0=BA=D1=81=D0=BE?= =?UTF-8?q?=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/services/classify.py | 28 ++++++++++----- deploy/services/metrics.py | 72 +++++++++++++++++++++++++++++++++++++ deploy/services/process.py | 5 +++ 3 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 deploy/services/metrics.py diff --git a/deploy/services/classify.py b/deploy/services/classify.py index 1262c4c..e85c2a7 100644 --- a/deploy/services/classify.py +++ b/deploy/services/classify.py @@ -56,7 +56,7 @@ def _is_garbage_by_header(text: str) -> bool: def _call_llm_classify(header_text): """ Прямой вызов LLM для классификации ОДНОГО документа. - Возвращает (parsed_dict, raw_text). + Возвращает (parsed_dict, raw_text, needed_fix). """ prompt, _ = build_classify_prompt(header_text) payload = { @@ -74,7 +74,8 @@ def _call_llm_classify(header_text): data = resp.json() raw = data.get("choices", [{}])[0].get("message", {}).get("content", "") - return _safe_json_parse(raw), raw + parsed, needed_fix = _safe_json_parse(raw) + return parsed, raw, needed_fix def classify_batch(batch_id): @@ -93,23 +94,32 @@ def classify_batch(batch_id): total = len(pending) done = 0 failed = 0 + garbage = 0 + json_fixes = 0 + json_total = 0 def _classify_one(doc): """Классифицировать один документ: фильтр → выжимка → LLM → сохранить.""" + nonlocal garbage, json_fixes, json_total try: # Stage 1: garbage by filename (0 tokens) if _is_garbage_by_filename(doc["filename"]): db_docs.set_classify_garbage(doc["id"], "filename_regex") + garbage += 1 return True # Stage 2: garbage by header keywords (0 tokens) text = _smart_extract(doc["elements_json"]) if _is_garbage_by_header(text): db_docs.set_classify_garbage(doc["id"], "header_keywords") + garbage += 1 return True # Stage 3: LLM classification (only for remaining) - result, raw = _call_llm_classify(text) + result, raw, needed_fix = _call_llm_classify(text) + json_total += 1 + if needed_fix: + json_fixes += 1 db_docs.set_classification( doc["id"], result.get("doc_type", "other"), @@ -133,11 +143,13 @@ def classify_batch(batch_id): else: failed += 1 - return {"ok": True, "total": total, "done": done, "failed": failed} + return {"ok": True, "total": total, "done": done, "failed": failed, + "garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3)} def _safe_json_parse(raw): - """Parse LLM response, fixing common JSON errors.""" + """Parse LLM response, fixing common JSON errors. + Returns (parsed_dict, needed_fix: bool).""" if not raw: raise ValueError("empty LLM response") @@ -156,7 +168,7 @@ def _safe_json_parse(raw): # Try strict parse try: - return json.loads(text) + return json.loads(text), False except json.JSONDecodeError: pass @@ -169,7 +181,7 @@ def _safe_json_parse(raw): # Try again try: - return json.loads(text) + return json.loads(text), True except json.JSONDecodeError: pass @@ -185,7 +197,7 @@ def _safe_json_parse(raw): text += '"' text += "}" - return json.loads(text) + return json.loads(text), True def _smart_extract(elements_json): diff --git a/deploy/services/metrics.py b/deploy/services/metrics.py new file mode 100644 index 0000000..9a37f22 --- /dev/null +++ b/deploy/services/metrics.py @@ -0,0 +1,72 @@ +"""Metrics — quality signals without golden dataset. + +Checks that work immediately: +- Arithmetic: sum == price * qty (free signal of LLM/data errors) +- JSON fix rate: how often _safe_json_parse has to repair LLM output +""" +from decimal import Decimal, InvalidOperation + + +def check_arithmetic(ops: list) -> list[dict]: + """Check sum == price * qty for ADD/UPDATE operations. + Returns list of mismatches: [{action, name, price, qty, expected_sum, actual_sum, diff}] + """ + mismatches = [] + for op in ops: + action = op.get("action", "") + if action not in ("ADD", "UPDATE"): + continue + + row = op.get("new_row") or op.get("new_values") or {} + price = _to_decimal(row.get("price")) + qty = _to_decimal(row.get("qty")) + actual_sum = _to_decimal(row.get("sum")) + + if price is None or qty is None or actual_sum is None: + continue # can't check without all three + + expected = price * qty + if expected != actual_sum: + mismatches.append({ + "action": action, + "name": row.get("name", "")[:100], + "price": float(price), + "qty": float(qty), + "expected_sum": float(expected), + "actual_sum": float(actual_sum), + "diff": float(actual_sum - expected), + }) + return mismatches + + +class ClassifyMetrics: + """Track _safe_json_parse fix rate per batch.""" + def __init__(self): + self.total = 0 + self.fixes = 0 # how many times JSON needed repair + + def record(self, needed_fix: bool): + self.total += 1 + if needed_fix: + self.fixes += 1 + + @property + def fix_rate(self) -> float: + return self.fixes / self.total if self.total else 0.0 + + def summary(self) -> dict: + return { + "total_classifications": self.total, + "json_fixes": self.fixes, + "json_fix_rate": round(self.fix_rate, 3), + } + + +def _to_decimal(val) -> Decimal | None: + """Safe conversion to Decimal.""" + if val is None or val == "": + return None + try: + return Decimal(str(val)) + except (InvalidOperation, ValueError): + return None diff --git a/deploy/services/process.py b/deploy/services/process.py index f70e8ff..52e3f0d 100644 --- a/deploy/services/process.py +++ b/deploy/services/process.py @@ -1,6 +1,7 @@ """Process service — SSE pipeline: reset → supplements → LLM → apply.""" import json, time, threading from db import supplements, spec_current, spec_events +from .metrics import check_arithmetic def run_pipeline(contract_id, order_ids, sse_send, build_prompt_fn): @@ -108,6 +109,10 @@ def run_pipeline(contract_id, order_ids, sse_send, build_prompt_fn): summary = spec_events.apply_ops( contract_id, sid, s.get("document_id", ""), ops, prompt_id, llm_result ) + # Arithmetic quality check (free signal, no golden dataset needed) + arith_mismatches = check_arithmetic(ops) + if arith_mismatches: + summary["arithmetic_mismatches"] = len(arith_mismatches) sse_send({"type": "applied", "supplement_id": sid, "summary": summary, "ops": ops}) total_time = round(time.time() - t0, 1)