v1.0.162: auto-classify backend — classify/grouping services, 7 new columns, /classify-batch, /api/groups, /apply-groups

deploy/convert_server.py

@@ -44,6 +44,10 @@ class Handler(BaseHTTPRequestHandler):
             self._handle_app_js()
         elif parsed.path == "/api/supplements":
             self._handle_api_supplements(parsed)
+        elif parsed.path == "/api/groups":
+            self._handle_api_groups(parsed)
+        elif parsed.path == "/api/batch-progress":
+            self._handle_api_batch_progress(parsed)
         elif parsed.path.startswith("/api/documents/"):
             self._handle_api_document(parsed)
         else:
@@ -58,6 +62,10 @@ class Handler(BaseHTTPRequestHandler):
             self._handle_llm_ops()
         elif self.path == "/convert-doc":
             self._handle_convert_doc()
+        elif self.path == "/classify-batch":
+            self._handle_classify_batch()
+        elif self.path == "/apply-groups":
+            self._handle_apply_groups()
         else:
             self.send_error(404)
 
@@ -119,6 +127,57 @@ class Handler(BaseHTTPRequestHandler):
             "elements_json": doc.get("elements_json"),
         })
 
+    # ── /api/groups ?batch=X ─────────────────────────────────────────────
+
+    def _handle_api_groups(self, parsed):
+        params = parse_qs(parsed.query)
+        batch_id = params.get("batch", [None])[0]
+        if not batch_id:
+            self._json({"ok": False, "error": "batch required"}, 400)
+            return
+        from services.grouping import group_documents
+        result = group_documents(batch_id)
+        self._json(result)
+
+    # ── /api/batch-progress ?batch=X ─────────────────────────────────────
+
+    def _handle_api_batch_progress(self, parsed):
+        params = parse_qs(parsed.query)
+        batch_id = params.get("batch", [None])[0]
+        if not batch_id:
+            self._json({"ok": False, "error": "batch required"}, 400)
+            return
+        counts = db_documents.count_by_status(batch_id)
+        docs = db_documents.list_by_batch(batch_id)
+        self._json({"ok": True, "counts": counts, "total": len(docs)})
+
+    # ── POST /classify-batch ─────────────────────────────────────────────
+
+    def _handle_classify_batch(self):
+        length = int(self.headers.get("Content-Length", 0))
+        body = json.loads(self.rfile.read(length))
+        batch_id = body.get("batch_id")
+        if not batch_id:
+            self._json({"ok": False, "error": "batch_id required"}, 400)
+            return
+        from services.classify import classify_batch
+        result = classify_batch(batch_id)
+        self._json(result)
+
+    # ── POST /apply-groups ───────────────────────────────────────────────
+
+    def _handle_apply_groups(self):
+        length = int(self.headers.get("Content-Length", 0))
+        body = json.loads(self.rfile.read(length))
+        batch_id = body.get("batch_id")
+        groups = body.get("groups", [])
+        if not batch_id:
+            self._json({"ok": False, "error": "batch_id required"}, 400)
+            return
+        from services.grouping import apply_groups
+        result = apply_groups(batch_id, groups)
+        self._json(result)
+
     # ── /app.js (static) ───────────────────────────────────────────────────
 
     def _handle_app_js(self):

deploy/db/documents.py

@@ -2,12 +2,12 @@
 from .connection import query, execute, execute_returning
 
 
-def insert(filename, mime_type, original_bytes, status="uploaded"):
+def insert(filename, mime_type, original_bytes, status="uploaded", batch_id=None):
     """Insert document, return row dict."""
     return execute_returning(
-        """INSERT INTO documents (filename, mime_type, original_bytes, status)
+        """INSERT INTO documents (filename, mime_type, original_bytes, status, batch_id)
-           VALUES (%s, %s, %s, %s) RETURNING *""",
+           VALUES (%s, %s, %s, %s, %s) RETURNING *""",
-        (filename, mime_type, original_bytes, status),
+        (filename, mime_type, original_bytes, status, batch_id),
     )
 
 
@@ -34,3 +34,47 @@ def set_error(doc_id, error_message):
 
 def delete(doc_id):
     return execute("DELETE FROM documents WHERE id = %s", (doc_id,))
+
+
+def set_classification(doc_id, doc_type, own_number, parent_number, doc_date, counterparty):
+    """Store LLM classification results."""
+    return execute(
+        """UPDATE documents SET doc_type=%s, own_number=%s, parent_number=%s,
+           doc_date=%s, counterparty=%s, classify_status='classified'
+           WHERE id=%s""",
+        (doc_type, own_number, parent_number, doc_date, counterparty, doc_id),
+    )
+
+
+def set_classify_failed(doc_id, error):
+    return execute(
+        "UPDATE documents SET classify_status='failed', error_message=%s WHERE id=%s",
+        (error, doc_id),
+    )
+
+
+def list_pending(batch_id):
+    """Documents waiting for classification."""
+    return query(
+        "SELECT id, filename, elements_json FROM documents WHERE batch_id=%s AND classify_status='pending' AND status='parsed'",
+        (batch_id,),
+    )
+
+
+def list_by_batch(batch_id):
+    """All documents in a batch with classification fields."""
+    return query(
+        """SELECT id, filename, status, doc_type, own_number, parent_number,
+           doc_date, counterparty, classify_status, error_message
+           FROM documents WHERE batch_id=%s ORDER BY created_at""",
+        (batch_id,),
+    )
+
+
+def count_by_status(batch_id):
+    """Count documents by classify_status."""
+    rows = query(
+        "SELECT classify_status, count(*) as cnt FROM documents WHERE batch_id=%s GROUP BY classify_status",
+        (batch_id,),
+    )
+    return {r["classify_status"]: r["cnt"] for r in rows}

deploy/db/prompts.py

@@ -35,3 +35,18 @@ def seed_defaults():
             true, 'Авто-создан из llm_prompt.py _build_diff'
         )
     """)
+    _ensure_classify_prompt()
+
+
+def _ensure_classify_prompt():
+    """Ensure classify prompt exists (idempotent)."""
+    existing = query("SELECT id FROM prompts WHERE role = 'classify' AND is_active = true LIMIT 1")
+    if existing:
+        return
+    execute("""
+        INSERT INTO prompts (role, name, body, is_active, notes) VALUES (
+            'classify', 'default-v1',
+            'Ты — классификатор договорных документов. Ниже фрагмент текста документа.\n\nОпредели:\n1. doc_type: "contract" (договор), "supplement" (допсоглашение/допник), "specification" (спецификация/приложение), "other"\n2. own_number: номер ЭТОГО документа (например "МЭС-123-2024" или "ДС №2")\n3. parent_number: номер родительского договора (для допников/спецификаций — "к Договору №..."). Для самого договора — null.\n4. doc_date: дата документа в формате YYYY-MM-DD\n5. counterparty: название контрагента (из поля "Заказчик", "Арендатор" или аналогичного)\n6. confidence: "ok" если уверен, "low" если сомневаешься\n\nВерни СТРОГО JSON без пояснений:\n{\n  "doc_type": "...",\n  "own_number": "..." или null,\n  "parent_number": "..." или null,\n  "doc_date": "YYYY-MM-DD" или null,\n  "counterparty": "..." или null,\n  "confidence": "ok" или "low"\n}\n\nДОКУМЕНТ:\n---\n{header_text}\n---',
+            true, 'Авто-создан: classify prompt'
+        )
+    """)

deploy/llm_prompt.py

@@ -232,3 +232,30 @@ def build_prompt(current_spec: list, doc_text: str) -> tuple:
     result = result.replace("{spec_current}", _build_spec_text(current_spec))
 
     return result, prompt_id
+
+
+def build_classify_prompt(header_text):
+    """Build classify prompt. Returns (prompt, prompt_id)."""
+    from db import prompts as db_prompts
+    prompt = db_prompts.get_active("classify")
+    if prompt:
+        body = prompt["body"].replace("{header_text}", header_text)
+        return body, prompt.get("id", "")
+    # Fallback
+    body = """Ты — классификатор договорных документов. Ниже фрагмент текста документа.
+
+Определи:
+1. doc_type: "contract", "supplement", "specification", "other"
+2. own_number: номер ЭТОГО документа
+3. parent_number: номер родительского договора (для допников/спецификаций)
+4. doc_date: дата в формате YYYY-MM-DD
+5. counterparty: название контрагента
+6. confidence: "ok" или "low"
+
+Верни СТРОГО JSON: {"doc_type":"...","own_number":"...","parent_number":"...","doc_date":"...","counterparty":"...","confidence":"..."}
+
+ДОКУМЕНТ:
+---
+{header_text}
+---""".replace("{header_text}", header_text)
+    return body, ""

deploy/services/classify.py

@@ -0,0 +1,124 @@
+"""Classify service — LLM-based document classification."""
+import json, re, os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import httpx
+from db import documents as db_docs
+from llm_prompt import build_classify_prompt
+
+log = __import__("logging").getLogger(__name__)
+
+MAX_WORKERS = 4
+LLM_URL = "https://api.aillm.ru/v1/chat/completions"
+LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "")
+LLM_MODEL = "gpt-oss-120b"
+
+
+def _call_llm_classify(header_text):
+    """Call LLM for classification. Returns parsed JSON dict."""
+    prompt, _ = build_classify_prompt(header_text)
+    payload = {
+        "model": LLM_MODEL,
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": 500,
+        "temperature": 0.1,
+    }
+    with httpx.Client(http2=True, timeout=60, verify=False) as client:
+        resp = client.post(
+            LLM_URL, json=payload,
+            headers={"Authorization": f"Bearer {LLM_KEY}", "Content-Type": "application/json"},
+        )
+        resp.raise_for_status()
+        data = resp.json()
+
+    raw = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+    # Extract JSON from possible markdown wrapping
+    json_text = raw
+    if "```json" in json_text:
+        json_text = json_text.split("```json")[1].split("```")[0]
+    elif "```" in json_text:
+        json_text = json_text.split("```")[1].split("```")[0]
+    return json.loads(json_text.strip())
+
+
+def classify_batch(batch_id):
+    """Classify all pending documents in a batch. Returns summary dict."""
+    pending = db_docs.list_pending(batch_id)
+    if not pending:
+        return {"ok": False, "error": "no pending documents"}
+
+    total = len(pending)
+    done = 0
+    failed = 0
+
+    def _classify_one(doc):
+        try:
+            text = _smart_extract(doc["elements_json"])
+            result = _call_llm_classify(text)
+            db_docs.set_classification(
+                doc["id"],
+                result.get("doc_type", "other"),
+                result.get("own_number"),
+                result.get("parent_number"),
+                result.get("doc_date"),
+                result.get("counterparty"),
+            )
+            return True
+        except Exception as e:
+            db_docs.set_classify_failed(doc["id"], str(e))
+            return False
+
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
+        futures = {pool.submit(_classify_one, d): d for d in pending}
+        for f in as_completed(futures):
+            if f.result():
+                done += 1
+            else:
+                failed += 1
+
+    return {"ok": True, "total": total, "done": done, "failed": failed}
+
+
+def _smart_extract(elements_json):
+    """Extract smart header: first ~1500 chars + marker lines from full doc."""
+    if not elements_json:
+        return ""
+
+    if isinstance(elements_json, str):
+        try:
+            elements = json.loads(elements_json)
+        except json.JSONDecodeError:
+            return elements_json[:2000]
+    elif isinstance(elements_json, list):
+        elements = elements_json
+    else:
+        return str(elements_json)[:2000]
+
+    # Build full text
+    lines = []
+    for el in elements:
+        if isinstance(el, dict):
+            t = el.get("type") or el.get("TYPE", "")
+            if t == "paragraph":
+                txt = el.get("text") or el.get("TEXT", "")
+                if txt:
+                    lines.append(txt)
+            elif t == "table":
+                rows = el.get("rows") or el.get("ROWS", [])
+                for row in rows:
+                    lines.append(" | ".join(str(c) for c in row))
+
+    full_text = "\n".join(lines)
+
+    # Header: first ~1500 chars
+    header = full_text[:1500]
+
+    # Marker lines: grep for key patterns
+    markers = re.findall(
+        r'.{0,200}(?:договор|№|соглашен|приложен|специф|контрагент|заказчик|арендатор).{0,200}',
+        full_text, re.IGNORECASE,
+    )
+    unique_markers = list(dict.fromkeys(markers))[:10]
+
+    combined = header + "\n---\n" + "\n".join(unique_markers)
+    return combined[:3000]

deploy/services/grouping.py

@@ -0,0 +1,95 @@
+"""Grouping service — match classified documents into contract groups."""
+import re
+from db import documents as db_docs
+from db import contracts as db_contracts
+from db import supplements as db_supplements
+
+
+def normalize_number(num):
+    """Normalize contract number for matching: uppercase, only letters/digits."""
+    if not num:
+        return ""
+    return re.sub(r"[^A-Z0-9А-Я]", "", num.upper())
+
+
+def group_documents(batch_id):
+    """Group classified documents by contract. Returns list of groups."""
+    docs = db_docs.list_by_batch(batch_id)
+    classified = [d for d in docs if d.get("classify_status") == "classified"]
+
+    # Separate contracts and supplements
+    contracts_list = []
+    supplements_list = []
+
+    for d in classified:
+        if d.get("doc_type") == "contract":
+            contracts_list.append(d)
+        else:
+            supplements_list.append(d)
+
+    groups = []
+
+    # Each contract becomes a group
+    for c in contracts_list:
+        group = {
+            "contract_number": c.get("own_number") or c.get("filename", ""),
+            "counterparty": c.get("counterparty") or "",
+            "documents": [c],
+        }
+        # Find supplements matching this contract
+        c_norm = normalize_number(c.get("own_number"))
+        for s in supplements_list:
+            parent = normalize_number(s.get("parent_number") or "")
+            own = normalize_number(s.get("own_number") or "")
+            if parent == c_norm or own == c_norm:
+                if s not in group["documents"]:
+                    group["documents"].append(s)
+
+        # Sort by date
+        group["documents"].sort(key=lambda x: x.get("doc_date") or "")
+
+        # Remove matched supplements from the pool
+        for s in group["documents"]:
+            if s in supplements_list:
+                supplements_list.remove(s)
+
+        groups.append(group)
+
+    # Unmatched documents → "unresolved" group
+    unmatched = [s for s in supplements_list if s.get("doc_type") != "contract"]
+    unmatched += [d for d in classified if d.get("classify_status") != "classified"]
+
+    if unmatched or len(contracts_list) == 0:
+        # Put unmatched into a separate group
+        if unmatched:
+            groups.append({
+                "contract_number": "__unresolved__",
+                "counterparty": "",
+                "documents": unmatched,
+            })
+
+    return {"ok": True, "groups": groups, "total_docs": len(classified)}
+
+
+def apply_groups(batch_id, groups_data):
+    """Apply confirmed groups: create contracts + supplements."""
+    created = 0
+    for g in groups_data:
+        contract_number = g.get("contract_number", "")
+        if contract_number == "__unresolved__":
+            continue
+        counterparty = g.get("counterparty", "")
+        docs = g.get("documents", [])
+
+        # Create contract
+        c = db_contracts.insert(contract_number, counterparty)
+        contract_id = c["id"]
+
+        # Create supplements in order
+        for i, d in enumerate(docs):
+            doc_id = d.get("id")
+            supp_type = "initial" if i == 0 else "additional"
+            db_supplements.insert(contract_id, doc_id, supp_type)
+            created += 1
+
+    return {"ok": True, "created": created}

deploy/services/upload.py

@@ -31,6 +31,8 @@ def handle_upload(rfile, content_type, content_length):
     if "contract_id" in fs:
         contract_id = fs.getfirst("contract_id", "")
 
+    batch_id = fs.getfirst("batch_id", None) if "batch_id" in fs else None
+
     if not filename or not file_data:
         return {"ok": False, "error": "no file in request"}
 
@@ -40,7 +42,7 @@ def handle_upload(rfile, content_type, content_length):
     if contract_id:
         supplements.delete_by_document(contract_id, filename)
 
-    doc = documents.insert(filename, mime, b64)
+    doc = documents.insert(filename, mime, b64, batch_id=batch_id)
 
     if not contract_id:
         from datetime import datetime

index.cfm

@@ -75,7 +75,7 @@
 <body>
 <div class="topbar">
   <img src="/nubes-logo.svg" alt="Nubes">
-  <span class="title">Сверка договоров — LLM AI-driven Event Sourcing <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.0.161 — Lucee</span></span>
+  <span class="title">Сверка договоров — LLM AI-driven Event Sourcing <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.0.162 — Lucee</span></span>
 </div>
 
 <div class="content">