feat: метрики качества — арифметика (sum==price*qty) + частота JSON-фиксов

deploy/services/classify.py

@@ -56,7 +56,7 @@ def _is_garbage_by_header(text: str) -> bool:
 def _call_llm_classify(header_text):
     """
     Прямой вызов LLM для классификации ОДНОГО документа.
-    Возвращает (parsed_dict, raw_text).
+    Возвращает (parsed_dict, raw_text, needed_fix).
     """
     prompt, _ = build_classify_prompt(header_text)
     payload = {
@@ -74,7 +74,8 @@ def _call_llm_classify(header_text):
         data = resp.json()
 
     raw = data.get("choices", [{}])[0].get("message", {}).get("content", "")
-    return _safe_json_parse(raw), raw
+    parsed, needed_fix = _safe_json_parse(raw)
+    return parsed, raw, needed_fix
 
 
 def classify_batch(batch_id):
@@ -93,23 +94,32 @@ def classify_batch(batch_id):
     total = len(pending)
     done = 0
     failed = 0
+    garbage = 0
+    json_fixes = 0
+    json_total = 0
 
     def _classify_one(doc):
         """Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
+        nonlocal garbage, json_fixes, json_total
         try:
             # Stage 1: garbage by filename (0 tokens)
             if _is_garbage_by_filename(doc["filename"]):
                 db_docs.set_classify_garbage(doc["id"], "filename_regex")
+                garbage += 1
                 return True
 
             # Stage 2: garbage by header keywords (0 tokens)
             text = _smart_extract(doc["elements_json"])
             if _is_garbage_by_header(text):
                 db_docs.set_classify_garbage(doc["id"], "header_keywords")
+                garbage += 1
                 return True
 
             # Stage 3: LLM classification (only for remaining)
-            result, raw = _call_llm_classify(text)
+            result, raw, needed_fix = _call_llm_classify(text)
+            json_total += 1
+            if needed_fix:
+                json_fixes += 1
             db_docs.set_classification(
                 doc["id"],
                 result.get("doc_type", "other"),
@@ -133,11 +143,13 @@ def classify_batch(batch_id):
             else:
                 failed += 1
 
-    return {"ok": True, "total": total, "done": done, "failed": failed}
+    return {"ok": True, "total": total, "done": done, "failed": failed,
+            "garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3)}
 
 
 def _safe_json_parse(raw):
-    """Parse LLM response, fixing common JSON errors."""
+    """Parse LLM response, fixing common JSON errors.
+    Returns (parsed_dict, needed_fix: bool)."""
     if not raw:
         raise ValueError("empty LLM response")
 
@@ -156,7 +168,7 @@ def _safe_json_parse(raw):
 
     # Try strict parse
     try:
-        return json.loads(text)
+        return json.loads(text), False
     except json.JSONDecodeError:
         pass
 
@@ -169,7 +181,7 @@ def _safe_json_parse(raw):
 
     # Try again
     try:
-        return json.loads(text)
+        return json.loads(text), True
     except json.JSONDecodeError:
         pass
 
@@ -185,7 +197,7 @@ def _safe_json_parse(raw):
             text += '"'
         text += "}"
 
-    return json.loads(text)
+    return json.loads(text), True
 
 
 def _smart_extract(elements_json):

deploy/services/metrics.py

@@ -0,0 +1,72 @@
+"""Metrics — quality signals without golden dataset.
+
+Checks that work immediately:
+- Arithmetic: sum == price * qty (free signal of LLM/data errors)
+- JSON fix rate: how often _safe_json_parse has to repair LLM output
+"""
+from decimal import Decimal, InvalidOperation
+
+
+def check_arithmetic(ops: list) -> list[dict]:
+    """Check sum == price * qty for ADD/UPDATE operations.
+    Returns list of mismatches: [{action, name, price, qty, expected_sum, actual_sum, diff}]
+    """
+    mismatches = []
+    for op in ops:
+        action = op.get("action", "")
+        if action not in ("ADD", "UPDATE"):
+            continue
+
+        row = op.get("new_row") or op.get("new_values") or {}
+        price = _to_decimal(row.get("price"))
+        qty = _to_decimal(row.get("qty"))
+        actual_sum = _to_decimal(row.get("sum"))
+
+        if price is None or qty is None or actual_sum is None:
+            continue  # can't check without all three
+
+        expected = price * qty
+        if expected != actual_sum:
+            mismatches.append({
+                "action": action,
+                "name": row.get("name", "")[:100],
+                "price": float(price),
+                "qty": float(qty),
+                "expected_sum": float(expected),
+                "actual_sum": float(actual_sum),
+                "diff": float(actual_sum - expected),
+            })
+    return mismatches
+
+
+class ClassifyMetrics:
+    """Track _safe_json_parse fix rate per batch."""
+    def __init__(self):
+        self.total = 0
+        self.fixes = 0  # how many times JSON needed repair
+
+    def record(self, needed_fix: bool):
+        self.total += 1
+        if needed_fix:
+            self.fixes += 1
+
+    @property
+    def fix_rate(self) -> float:
+        return self.fixes / self.total if self.total else 0.0
+
+    def summary(self) -> dict:
+        return {
+            "total_classifications": self.total,
+            "json_fixes": self.fixes,
+            "json_fix_rate": round(self.fix_rate, 3),
+        }
+
+
+def _to_decimal(val) -> Decimal | None:
+    """Safe conversion to Decimal."""
+    if val is None or val == "":
+        return None
+    try:
+        return Decimal(str(val))
+    except (InvalidOperation, ValueError):
+        return None

deploy/services/process.py

@@ -1,6 +1,7 @@
 """Process service — SSE pipeline: reset → supplements → LLM → apply."""
 import json, time, threading
 from db import supplements, spec_current, spec_events
+from .metrics import check_arithmetic
 
 
 def run_pipeline(contract_id, order_ids, sse_send, build_prompt_fn):
@@ -108,6 +109,10 @@ def run_pipeline(contract_id, order_ids, sse_send, build_prompt_fn):
         summary = spec_events.apply_ops(
             contract_id, sid, s.get("document_id", ""), ops, prompt_id, llm_result
         )
+        # Arithmetic quality check (free signal, no golden dataset needed)
+        arith_mismatches = check_arithmetic(ops)
+        if arith_mismatches:
+            summary["arithmetic_mismatches"] = len(arith_mismatches)
         sse_send({"type": "applied", "supplement_id": sid, "summary": summary, "ops": ops})
 
     total_time = round(time.time() - t0, 1)