v1.0.162: auto-classify backend — classify/grouping services, 7 new columns, /classify-batch, /api/groups, /apply-groups

deploy/services/classify.py

@@ -0,0 +1,124 @@
+"""Classify service — LLM-based document classification."""
+import json, re, os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import httpx
+from db import documents as db_docs
+from llm_prompt import build_classify_prompt
+
+log = __import__("logging").getLogger(__name__)
+
+MAX_WORKERS = 4
+LLM_URL = "https://api.aillm.ru/v1/chat/completions"
+LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "")
+LLM_MODEL = "gpt-oss-120b"
+
+
+def _call_llm_classify(header_text):
+    """Call LLM for classification. Returns parsed JSON dict."""
+    prompt, _ = build_classify_prompt(header_text)
+    payload = {
+        "model": LLM_MODEL,
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": 500,
+        "temperature": 0.1,
+    }
+    with httpx.Client(http2=True, timeout=60, verify=False) as client:
+        resp = client.post(
+            LLM_URL, json=payload,
+            headers={"Authorization": f"Bearer {LLM_KEY}", "Content-Type": "application/json"},
+        )
+        resp.raise_for_status()
+        data = resp.json()
+
+    raw = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+    # Extract JSON from possible markdown wrapping
+    json_text = raw
+    if "```json" in json_text:
+        json_text = json_text.split("```json")[1].split("```")[0]
+    elif "```" in json_text:
+        json_text = json_text.split("```")[1].split("```")[0]
+    return json.loads(json_text.strip())
+
+
+def classify_batch(batch_id):
+    """Classify all pending documents in a batch. Returns summary dict."""
+    pending = db_docs.list_pending(batch_id)
+    if not pending:
+        return {"ok": False, "error": "no pending documents"}
+
+    total = len(pending)
+    done = 0
+    failed = 0
+
+    def _classify_one(doc):
+        try:
+            text = _smart_extract(doc["elements_json"])
+            result = _call_llm_classify(text)
+            db_docs.set_classification(
+                doc["id"],
+                result.get("doc_type", "other"),
+                result.get("own_number"),
+                result.get("parent_number"),
+                result.get("doc_date"),
+                result.get("counterparty"),
+            )
+            return True
+        except Exception as e:
+            db_docs.set_classify_failed(doc["id"], str(e))
+            return False
+
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
+        futures = {pool.submit(_classify_one, d): d for d in pending}
+        for f in as_completed(futures):
+            if f.result():
+                done += 1
+            else:
+                failed += 1
+
+    return {"ok": True, "total": total, "done": done, "failed": failed}
+
+
+def _smart_extract(elements_json):
+    """Extract smart header: first ~1500 chars + marker lines from full doc."""
+    if not elements_json:
+        return ""
+
+    if isinstance(elements_json, str):
+        try:
+            elements = json.loads(elements_json)
+        except json.JSONDecodeError:
+            return elements_json[:2000]
+    elif isinstance(elements_json, list):
+        elements = elements_json
+    else:
+        return str(elements_json)[:2000]
+
+    # Build full text
+    lines = []
+    for el in elements:
+        if isinstance(el, dict):
+            t = el.get("type") or el.get("TYPE", "")
+            if t == "paragraph":
+                txt = el.get("text") or el.get("TEXT", "")
+                if txt:
+                    lines.append(txt)
+            elif t == "table":
+                rows = el.get("rows") or el.get("ROWS", [])
+                for row in rows:
+                    lines.append(" | ".join(str(c) for c in row))
+
+    full_text = "\n".join(lines)
+
+    # Header: first ~1500 chars
+    header = full_text[:1500]
+
+    # Marker lines: grep for key patterns
+    markers = re.findall(
+        r'.{0,200}(?:договор|№|соглашен|приложен|специф|контрагент|заказчик|арендатор).{0,200}',
+        full_text, re.IGNORECASE,
+    )
+    unique_markers = list(dict.fromkeys(markers))[:10]
+
+    combined = header + "\n---\n" + "\n".join(unique_markers)
+    return combined[:3000]

deploy/services/grouping.py

@@ -0,0 +1,95 @@
+"""Grouping service — match classified documents into contract groups."""
+import re
+from db import documents as db_docs
+from db import contracts as db_contracts
+from db import supplements as db_supplements
+
+
+def normalize_number(num):
+    """Normalize contract number for matching: uppercase, only letters/digits."""
+    if not num:
+        return ""
+    return re.sub(r"[^A-Z0-9А-Я]", "", num.upper())
+
+
+def group_documents(batch_id):
+    """Group classified documents by contract. Returns list of groups."""
+    docs = db_docs.list_by_batch(batch_id)
+    classified = [d for d in docs if d.get("classify_status") == "classified"]
+
+    # Separate contracts and supplements
+    contracts_list = []
+    supplements_list = []
+
+    for d in classified:
+        if d.get("doc_type") == "contract":
+            contracts_list.append(d)
+        else:
+            supplements_list.append(d)
+
+    groups = []
+
+    # Each contract becomes a group
+    for c in contracts_list:
+        group = {
+            "contract_number": c.get("own_number") or c.get("filename", ""),
+            "counterparty": c.get("counterparty") or "",
+            "documents": [c],
+        }
+        # Find supplements matching this contract
+        c_norm = normalize_number(c.get("own_number"))
+        for s in supplements_list:
+            parent = normalize_number(s.get("parent_number") or "")
+            own = normalize_number(s.get("own_number") or "")
+            if parent == c_norm or own == c_norm:
+                if s not in group["documents"]:
+                    group["documents"].append(s)
+
+        # Sort by date
+        group["documents"].sort(key=lambda x: x.get("doc_date") or "")
+
+        # Remove matched supplements from the pool
+        for s in group["documents"]:
+            if s in supplements_list:
+                supplements_list.remove(s)
+
+        groups.append(group)
+
+    # Unmatched documents → "unresolved" group
+    unmatched = [s for s in supplements_list if s.get("doc_type") != "contract"]
+    unmatched += [d for d in classified if d.get("classify_status") != "classified"]
+
+    if unmatched or len(contracts_list) == 0:
+        # Put unmatched into a separate group
+        if unmatched:
+            groups.append({
+                "contract_number": "__unresolved__",
+                "counterparty": "",
+                "documents": unmatched,
+            })
+
+    return {"ok": True, "groups": groups, "total_docs": len(classified)}
+
+
+def apply_groups(batch_id, groups_data):
+    """Apply confirmed groups: create contracts + supplements."""
+    created = 0
+    for g in groups_data:
+        contract_number = g.get("contract_number", "")
+        if contract_number == "__unresolved__":
+            continue
+        counterparty = g.get("counterparty", "")
+        docs = g.get("documents", [])
+
+        # Create contract
+        c = db_contracts.insert(contract_number, counterparty)
+        contract_id = c["id"]
+
+        # Create supplements in order
+        for i, d in enumerate(docs):
+            doc_id = d.get("id")
+            supp_type = "initial" if i == 0 else "additional"
+            db_supplements.insert(contract_id, doc_id, supp_type)
+            created += 1
+
+    return {"ok": True, "created": created}

deploy/services/upload.py

@@ -31,6 +31,8 @@ def handle_upload(rfile, content_type, content_length):
     if "contract_id" in fs:
         contract_id = fs.getfirst("contract_id", "")
 
+    batch_id = fs.getfirst("batch_id", None) if "batch_id" in fs else None
+
     if not filename or not file_data:
         return {"ok": False, "error": "no file in request"}
 
@@ -40,7 +42,7 @@ def handle_upload(rfile, content_type, content_length):
     if contract_id:
         supplements.delete_by_document(contract_id, filename)
 
-    doc = documents.insert(filename, mime, b64)
+    doc = documents.insert(filename, mime, b64, batch_id=batch_id)
 
     if not contract_id:
         from datetime import datetime