diff --git a/deploy/convert_server.py b/deploy/convert_server.py
index c47b12f..3e408b4 100755
--- a/deploy/convert_server.py
+++ b/deploy/convert_server.py
@@ -44,6 +44,10 @@ class Handler(BaseHTTPRequestHandler):
self._handle_app_js()
elif parsed.path == "/api/supplements":
self._handle_api_supplements(parsed)
+ elif parsed.path == "/api/groups":
+ self._handle_api_groups(parsed)
+ elif parsed.path == "/api/batch-progress":
+ self._handle_api_batch_progress(parsed)
elif parsed.path.startswith("/api/documents/"):
self._handle_api_document(parsed)
else:
@@ -58,6 +62,10 @@ class Handler(BaseHTTPRequestHandler):
self._handle_llm_ops()
elif self.path == "/convert-doc":
self._handle_convert_doc()
+ elif self.path == "/classify-batch":
+ self._handle_classify_batch()
+ elif self.path == "/apply-groups":
+ self._handle_apply_groups()
else:
self.send_error(404)
@@ -119,6 +127,57 @@ class Handler(BaseHTTPRequestHandler):
"elements_json": doc.get("elements_json"),
})
+ # ── /api/groups ?batch=X ─────────────────────────────────────────────
+
+ def _handle_api_groups(self, parsed):
+ params = parse_qs(parsed.query)
+ batch_id = params.get("batch", [None])[0]
+ if not batch_id:
+ self._json({"ok": False, "error": "batch required"}, 400)
+ return
+ from services.grouping import group_documents
+ result = group_documents(batch_id)
+ self._json(result)
+
+ # ── /api/batch-progress ?batch=X ─────────────────────────────────────
+
+ def _handle_api_batch_progress(self, parsed):
+ params = parse_qs(parsed.query)
+ batch_id = params.get("batch", [None])[0]
+ if not batch_id:
+ self._json({"ok": False, "error": "batch required"}, 400)
+ return
+ counts = db_documents.count_by_status(batch_id)
+ docs = db_documents.list_by_batch(batch_id)
+ self._json({"ok": True, "counts": counts, "total": len(docs)})
+
+ # ── POST /classify-batch ─────────────────────────────────────────────
+
+ def _handle_classify_batch(self):
+ length = int(self.headers.get("Content-Length", 0))
+ body = json.loads(self.rfile.read(length))
+ batch_id = body.get("batch_id")
+ if not batch_id:
+ self._json({"ok": False, "error": "batch_id required"}, 400)
+ return
+ from services.classify import classify_batch
+ result = classify_batch(batch_id)
+ self._json(result)
+
+ # ── POST /apply-groups ───────────────────────────────────────────────
+
+ def _handle_apply_groups(self):
+ length = int(self.headers.get("Content-Length", 0))
+ body = json.loads(self.rfile.read(length))
+ batch_id = body.get("batch_id")
+ groups = body.get("groups", [])
+ if not batch_id:
+ self._json({"ok": False, "error": "batch_id required"}, 400)
+ return
+ from services.grouping import apply_groups
+ result = apply_groups(batch_id, groups)
+ self._json(result)
+
# ── /app.js (static) ───────────────────────────────────────────────────
def _handle_app_js(self):
diff --git a/deploy/db/documents.py b/deploy/db/documents.py
index e3fa211..5068932 100644
--- a/deploy/db/documents.py
+++ b/deploy/db/documents.py
@@ -2,12 +2,12 @@
from .connection import query, execute, execute_returning
-def insert(filename, mime_type, original_bytes, status="uploaded"):
+def insert(filename, mime_type, original_bytes, status="uploaded", batch_id=None):
"""Insert document, return row dict."""
return execute_returning(
- """INSERT INTO documents (filename, mime_type, original_bytes, status)
- VALUES (%s, %s, %s, %s) RETURNING *""",
- (filename, mime_type, original_bytes, status),
+ """INSERT INTO documents (filename, mime_type, original_bytes, status, batch_id)
+ VALUES (%s, %s, %s, %s, %s) RETURNING *""",
+ (filename, mime_type, original_bytes, status, batch_id),
)
@@ -34,3 +34,47 @@ def set_error(doc_id, error_message):
def delete(doc_id):
return execute("DELETE FROM documents WHERE id = %s", (doc_id,))
+
+
+def set_classification(doc_id, doc_type, own_number, parent_number, doc_date, counterparty):
+ """Store LLM classification results."""
+ return execute(
+ """UPDATE documents SET doc_type=%s, own_number=%s, parent_number=%s,
+ doc_date=%s, counterparty=%s, classify_status='classified'
+ WHERE id=%s""",
+ (doc_type, own_number, parent_number, doc_date, counterparty, doc_id),
+ )
+
+
+def set_classify_failed(doc_id, error):
+ return execute(
+ "UPDATE documents SET classify_status='failed', error_message=%s WHERE id=%s",
+ (error, doc_id),
+ )
+
+
+def list_pending(batch_id):
+ """Documents waiting for classification."""
+ return query(
+ "SELECT id, filename, elements_json FROM documents WHERE batch_id=%s AND classify_status='pending' AND status='parsed'",
+ (batch_id,),
+ )
+
+
+def list_by_batch(batch_id):
+ """All documents in a batch with classification fields."""
+ return query(
+ """SELECT id, filename, status, doc_type, own_number, parent_number,
+ doc_date, counterparty, classify_status, error_message
+ FROM documents WHERE batch_id=%s ORDER BY created_at""",
+ (batch_id,),
+ )
+
+
+def count_by_status(batch_id):
+ """Count documents by classify_status."""
+ rows = query(
+ "SELECT classify_status, count(*) as cnt FROM documents WHERE batch_id=%s GROUP BY classify_status",
+ (batch_id,),
+ )
+ return {r["classify_status"]: r["cnt"] for r in rows}
diff --git a/deploy/db/prompts.py b/deploy/db/prompts.py
index 5209907..f9fc220 100644
--- a/deploy/db/prompts.py
+++ b/deploy/db/prompts.py
@@ -35,3 +35,18 @@ def seed_defaults():
true, 'Авто-создан из llm_prompt.py _build_diff'
)
""")
+ _ensure_classify_prompt()
+
+
+def _ensure_classify_prompt():
+ """Ensure classify prompt exists (idempotent)."""
+ existing = query("SELECT id FROM prompts WHERE role = 'classify' AND is_active = true LIMIT 1")
+ if existing:
+ return
+ execute("""
+ INSERT INTO prompts (role, name, body, is_active, notes) VALUES (
+ 'classify', 'default-v1',
+ 'Ты — классификатор договорных документов. Ниже фрагмент текста документа.\n\nОпредели:\n1. doc_type: "contract" (договор), "supplement" (допсоглашение/допник), "specification" (спецификация/приложение), "other"\n2. own_number: номер ЭТОГО документа (например "МЭС-123-2024" или "ДС №2")\n3. parent_number: номер родительского договора (для допников/спецификаций — "к Договору №..."). Для самого договора — null.\n4. doc_date: дата документа в формате YYYY-MM-DD\n5. counterparty: название контрагента (из поля "Заказчик", "Арендатор" или аналогичного)\n6. confidence: "ok" если уверен, "low" если сомневаешься\n\nВерни СТРОГО JSON без пояснений:\n{\n "doc_type": "...",\n "own_number": "..." или null,\n "parent_number": "..." или null,\n "doc_date": "YYYY-MM-DD" или null,\n "counterparty": "..." или null,\n "confidence": "ok" или "low"\n}\n\nДОКУМЕНТ:\n---\n{header_text}\n---',
+ true, 'Авто-создан: classify prompt'
+ )
+ """)
diff --git a/deploy/llm_prompt.py b/deploy/llm_prompt.py
index 58bae56..214046a 100644
--- a/deploy/llm_prompt.py
+++ b/deploy/llm_prompt.py
@@ -232,3 +232,30 @@ def build_prompt(current_spec: list, doc_text: str) -> tuple:
result = result.replace("{spec_current}", _build_spec_text(current_spec))
return result, prompt_id
+
+
+def build_classify_prompt(header_text):
+ """Build classify prompt. Returns (prompt, prompt_id)."""
+ from db import prompts as db_prompts
+ prompt = db_prompts.get_active("classify")
+ if prompt:
+ body = prompt["body"].replace("{header_text}", header_text)
+ return body, prompt.get("id", "")
+ # Fallback
+ body = """Ты — классификатор договорных документов. Ниже фрагмент текста документа.
+
+Определи:
+1. doc_type: "contract", "supplement", "specification", "other"
+2. own_number: номер ЭТОГО документа
+3. parent_number: номер родительского договора (для допников/спецификаций)
+4. doc_date: дата в формате YYYY-MM-DD
+5. counterparty: название контрагента
+6. confidence: "ok" или "low"
+
+Верни СТРОГО JSON: {"doc_type":"...","own_number":"...","parent_number":"...","doc_date":"...","counterparty":"...","confidence":"..."}
+
+ДОКУМЕНТ:
+---
+{header_text}
+---""".replace("{header_text}", header_text)
+ return body, ""
diff --git a/deploy/services/classify.py b/deploy/services/classify.py
new file mode 100644
index 0000000..0274d06
--- /dev/null
+++ b/deploy/services/classify.py
@@ -0,0 +1,124 @@
+"""Classify service — LLM-based document classification."""
+import json, re, os
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import httpx
+from db import documents as db_docs
+from llm_prompt import build_classify_prompt
+
+log = __import__("logging").getLogger(__name__)
+
+MAX_WORKERS = 4
+LLM_URL = "https://api.aillm.ru/v1/chat/completions"
+LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "")
+LLM_MODEL = "gpt-oss-120b"
+
+
+def _call_llm_classify(header_text):
+ """Call LLM for classification. Returns parsed JSON dict."""
+ prompt, _ = build_classify_prompt(header_text)
+ payload = {
+ "model": LLM_MODEL,
+ "messages": [{"role": "user", "content": prompt}],
+ "max_tokens": 500,
+ "temperature": 0.1,
+ }
+ with httpx.Client(http2=True, timeout=60, verify=False) as client:
+ resp = client.post(
+ LLM_URL, json=payload,
+ headers={"Authorization": f"Bearer {LLM_KEY}", "Content-Type": "application/json"},
+ )
+ resp.raise_for_status()
+ data = resp.json()
+
+ raw = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+ # Extract JSON from possible markdown wrapping
+ json_text = raw
+ if "```json" in json_text:
+ json_text = json_text.split("```json")[1].split("```")[0]
+ elif "```" in json_text:
+ json_text = json_text.split("```")[1].split("```")[0]
+ return json.loads(json_text.strip())
+
+
+def classify_batch(batch_id):
+ """Classify all pending documents in a batch. Returns summary dict."""
+ pending = db_docs.list_pending(batch_id)
+ if not pending:
+ return {"ok": False, "error": "no pending documents"}
+
+ total = len(pending)
+ done = 0
+ failed = 0
+
+ def _classify_one(doc):
+ try:
+ text = _smart_extract(doc["elements_json"])
+ result = _call_llm_classify(text)
+ db_docs.set_classification(
+ doc["id"],
+ result.get("doc_type", "other"),
+ result.get("own_number"),
+ result.get("parent_number"),
+ result.get("doc_date"),
+ result.get("counterparty"),
+ )
+ return True
+ except Exception as e:
+ db_docs.set_classify_failed(doc["id"], str(e))
+ return False
+
+ with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
+ futures = {pool.submit(_classify_one, d): d for d in pending}
+ for f in as_completed(futures):
+ if f.result():
+ done += 1
+ else:
+ failed += 1
+
+ return {"ok": True, "total": total, "done": done, "failed": failed}
+
+
+def _smart_extract(elements_json):
+ """Extract smart header: first ~1500 chars + marker lines from full doc."""
+ if not elements_json:
+ return ""
+
+ if isinstance(elements_json, str):
+ try:
+ elements = json.loads(elements_json)
+ except json.JSONDecodeError:
+ return elements_json[:2000]
+ elif isinstance(elements_json, list):
+ elements = elements_json
+ else:
+ return str(elements_json)[:2000]
+
+ # Build full text
+ lines = []
+ for el in elements:
+ if isinstance(el, dict):
+ t = el.get("type") or el.get("TYPE", "")
+ if t == "paragraph":
+ txt = el.get("text") or el.get("TEXT", "")
+ if txt:
+ lines.append(txt)
+ elif t == "table":
+ rows = el.get("rows") or el.get("ROWS", [])
+ for row in rows:
+ lines.append(" | ".join(str(c) for c in row))
+
+ full_text = "\n".join(lines)
+
+ # Header: first ~1500 chars
+ header = full_text[:1500]
+
+ # Marker lines: grep for key patterns
+ markers = re.findall(
+ r'.{0,200}(?:договор|№|соглашен|приложен|специф|контрагент|заказчик|арендатор).{0,200}',
+ full_text, re.IGNORECASE,
+ )
+ unique_markers = list(dict.fromkeys(markers))[:10]
+
+ combined = header + "\n---\n" + "\n".join(unique_markers)
+ return combined[:3000]
diff --git a/deploy/services/grouping.py b/deploy/services/grouping.py
new file mode 100644
index 0000000..af0e614
--- /dev/null
+++ b/deploy/services/grouping.py
@@ -0,0 +1,95 @@
+"""Grouping service — match classified documents into contract groups."""
+import re
+from db import documents as db_docs
+from db import contracts as db_contracts
+from db import supplements as db_supplements
+
+
+def normalize_number(num):
+ """Normalize contract number for matching: uppercase, only letters/digits."""
+ if not num:
+ return ""
+ return re.sub(r"[^A-Z0-9А-Я]", "", num.upper())
+
+
+def group_documents(batch_id):
+ """Group classified documents by contract. Returns list of groups."""
+ docs = db_docs.list_by_batch(batch_id)
+ classified = [d for d in docs if d.get("classify_status") == "classified"]
+
+ # Separate contracts and supplements
+ contracts_list = []
+ supplements_list = []
+
+ for d in classified:
+ if d.get("doc_type") == "contract":
+ contracts_list.append(d)
+ else:
+ supplements_list.append(d)
+
+ groups = []
+
+ # Each contract becomes a group
+ for c in contracts_list:
+ group = {
+ "contract_number": c.get("own_number") or c.get("filename", ""),
+ "counterparty": c.get("counterparty") or "",
+ "documents": [c],
+ }
+ # Find supplements matching this contract
+ c_norm = normalize_number(c.get("own_number"))
+ for s in supplements_list:
+ parent = normalize_number(s.get("parent_number") or "")
+ own = normalize_number(s.get("own_number") or "")
+ if parent == c_norm or own == c_norm:
+ if s not in group["documents"]:
+ group["documents"].append(s)
+
+ # Sort by date
+ group["documents"].sort(key=lambda x: x.get("doc_date") or "")
+
+ # Remove matched supplements from the pool
+ for s in group["documents"]:
+ if s in supplements_list:
+ supplements_list.remove(s)
+
+ groups.append(group)
+
+ # Unmatched documents → "unresolved" group
+ unmatched = [s for s in supplements_list if s.get("doc_type") != "contract"]
+ unmatched += [d for d in classified if d.get("classify_status") != "classified"]
+
+ if unmatched or len(contracts_list) == 0:
+ # Put unmatched into a separate group
+ if unmatched:
+ groups.append({
+ "contract_number": "__unresolved__",
+ "counterparty": "",
+ "documents": unmatched,
+ })
+
+ return {"ok": True, "groups": groups, "total_docs": len(classified)}
+
+
+def apply_groups(batch_id, groups_data):
+ """Apply confirmed groups: create contracts + supplements."""
+ created = 0
+ for g in groups_data:
+ contract_number = g.get("contract_number", "")
+ if contract_number == "__unresolved__":
+ continue
+ counterparty = g.get("counterparty", "")
+ docs = g.get("documents", [])
+
+ # Create contract
+ c = db_contracts.insert(contract_number, counterparty)
+ contract_id = c["id"]
+
+ # Create supplements in order
+ for i, d in enumerate(docs):
+ doc_id = d.get("id")
+ supp_type = "initial" if i == 0 else "additional"
+ db_supplements.insert(contract_id, doc_id, supp_type)
+ created += 1
+
+ return {"ok": True, "created": created}
diff --git a/deploy/services/upload.py b/deploy/services/upload.py
index 03faa5b..7b5c091 100644
--- a/deploy/services/upload.py
+++ b/deploy/services/upload.py
@@ -31,6 +31,8 @@ def handle_upload(rfile, content_type, content_length):
if "contract_id" in fs:
contract_id = fs.getfirst("contract_id", "")
+ batch_id = fs.getfirst("batch_id", None) if "batch_id" in fs else None
+
if not filename or not file_data:
return {"ok": False, "error": "no file in request"}
@@ -40,7 +42,7 @@ def handle_upload(rfile, content_type, content_length):
if contract_id:
supplements.delete_by_document(contract_id, filename)
- doc = documents.insert(filename, mime, b64)
+ doc = documents.insert(filename, mime, b64, batch_id=batch_id)
if not contract_id:
from datetime import datetime
diff --git a/index.cfm b/index.cfm
index 7462670..22ac895 100644
--- a/index.cfm
+++ b/index.cfm
@@ -75,7 +75,7 @@
-
Сверка договоров — LLM AI-driven Event Sourcing v1.0.161 — Lucee
+
Сверка договоров — LLM AI-driven Event Sourcing v1.0.162 — Lucee