From 475d0b8dc7699be09841d7ed9a73ea6c1ee747bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Wed, 24 Jun 2026 17:06:25 +0400 Subject: [PATCH] =?UTF-8?q?v1.0.175:=20=D0=B2=D0=B8=D1=80=D1=82=D1=83?= =?UTF-8?q?=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D0=B5=20=D0=B3=D1=80=D1=83=D0=BF?= =?UTF-8?q?=D0=BF=D1=8B=20+=20classify=5Fraw=20+=20stepper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy/app.js | 46 ++++++++++++++++++++++++++++++++++ deploy/app_utils.js | 2 ++ deploy/convert_server.py | 11 +++++++- deploy/db/documents.py | 10 ++++---- deploy/services/classify.py | 12 ++++----- deploy/services/grouping.py | 50 ++++++++++++++++++++++++++++--------- index.cfm | 8 +++++- 7 files changed, 114 insertions(+), 25 deletions(-) diff --git a/deploy/app.js b/deploy/app.js index bfc8587..fb24596 100644 --- a/deploy/app.js +++ b/deploy/app.js @@ -29,6 +29,26 @@ function syncDB() { body: JSON.stringify({keep_ids: keepIds}) }).catch(function(){}); } +// ── Pipeline stepper ───────────────────────────────────────── +function stepDone(id) { + var el = document.getElementById(id); + if (el) { el.innerHTML = el.innerHTML.replace('○','✓'); el.style.color = 'var(--green)'; } +} +function stepActive(id) { + var el = document.getElementById(id); + if (el) { el.innerHTML = el.innerHTML.replace('○','⏳').replace('✓','⏳'); el.style.color = 'var(--brand-primary)'; } +} +function resetStepper(fromId) { + var steps = ['stepUpload','stepClassify','stepGroups','stepCompare']; + var reset = false; + steps.forEach(function(id) { + if (id === fromId) reset = true; + if (reset) { + var el = document.getElementById(id); + if (el) { el.innerHTML = el.innerHTML.replace('✓','○').replace('⏳','○'); el.style.color = 'var(--muted)'; } + } + }); +} // Утилиты (formatSize, formatDate, moveUp, moveDown, escHtml, fmtDate) — // вынесены в app_utils.js для облегчения анализа и отладки. @@ -67,6 +87,8 @@ fileInput.addEventListener('change', async function() { renderTable(); // Разблокировать кнопку классификации — состав файлов изменился showClassifyBtn(); + // Сбросить прогресс пайплайна + resetStepper('stepUpload'); for (var i = 0; i < newFiles.length; i++) { var f = newFiles[i]; @@ -186,6 +208,8 @@ fileInput.addEventListener('change', async function() { } await refreshSupps(); + stepDone('stepUpload'); + stepActive('stepClassify'); // Синхронизировать БД с таблицей — удалить всё, чего нет в fileQueue syncDB(); @@ -220,6 +244,7 @@ async function runClassify() { var btn = document.getElementById('classifyBtn'); btn.disabled = true; btn.innerHTML = ' Классификация... 0с'; + stepActive('stepClassify'); var start = Date.now(); var timer = setInterval(function() { @@ -253,6 +278,8 @@ async function runClassify() { clearInterval(pollTimer); if (data.ok) { btn.innerHTML = '✓ ' + data.done + '/' + data.total + ' классифицировано (' + Math.round((Date.now() - start) / 1000) + 'с)'; + stepDone('stepClassify'); + stepActive('stepGroups'); loadGroups(); } else { btn.innerHTML = '✗ ' + (data.error || 'ошибка'); @@ -277,6 +304,7 @@ async function loadGroups() { var realGroups = data.groups.filter(function(g) { return g.contract_number !== '__unresolved__'; }); var unresolvedGroup = data.groups.find(function(g) { return g.contract_number === '__unresolved__'; }); diffBody.innerHTML = '
📋 Найдено групп: ' + realGroups.length + (unresolvedGroup ? ' | ⚠ ' + unresolvedGroup.documents.length + ' файлов не распознано' : '') + '
'; + stepDone('stepGroups'); // Сохраняем группы для compare window._groupsData = data.groups; @@ -346,6 +374,7 @@ window.runCompareForGroup = async function(gi) { var cid = ad.contract_ids[0]; btn.innerHTML = ' Сравнение...'; + stepActive('stepCompare'); var diffCard = document.getElementById('diffCard'); var diffBody = document.getElementById('diffBody'); @@ -539,6 +568,7 @@ document.getElementById('llmBtn').addEventListener('click', function() { es.close(); diffStatus.textContent = '✓ ' + d.total_time_s + 'с'; btn.innerHTML = ' ✓ Готово'; + stepDone('stepCompare'); if (d.total_unresolved > 0) { diffBody.innerHTML += '
⚠ ' + d.total_unresolved + ' неразрешённых строк — разобрать
'; } @@ -688,6 +718,22 @@ async function showText(i) { if (f.supp_id) { rightHtml += '
Тип ДС' + (f.supp_type || '—') + '
'; } + + // ── Результаты классификации ────────────────────────── + if (qData.classify_status === 'classified' || qData.classify_raw) { + rightHtml += '
🏷️ Классификация LLM
'; + if (qData.doc_type) rightHtml += '
Тип' + qData.doc_type + '
'; + if (qData.own_number) rightHtml += '
Номер' + qData.own_number + '
'; + if (qData.parent_number) rightHtml += '
Родительский' + qData.parent_number + '
'; + if (qData.doc_date) rightHtml += '
Дата' + qData.doc_date + '
'; + if (qData.counterparty) rightHtml += '
Контрагент' + qData.counterparty + '
'; + if (qData.classify_raw) { + var rawId = 'raw_' + docId.replace(/-/g,''); + rightHtml += '
Сырой ответ LLM'; + rightHtml += '
' + escHtml(qData.classify_raw) + '
'; + rightHtml += '
'; + } + } } catch(e) { leftHtml = 'Ошибка'; } diff --git a/deploy/app_utils.js b/deploy/app_utils.js index 1b55509..0a54d46 100644 --- a/deploy/app_utils.js +++ b/deploy/app_utils.js @@ -35,6 +35,8 @@ function removeFile(i) { renderTable(); // Синхронизировать БД с таблицей if (typeof syncDB === 'function') syncDB(); + // Сбросить прогресс при изменении состава файлов + if (typeof resetStepper === 'function') resetStepper('stepUpload'); // Разблокировать кнопку классификации при изменении состава файлов if (typeof showClassifyBtn === 'function') showClassifyBtn(); } diff --git a/deploy/convert_server.py b/deploy/convert_server.py index 006fa96..f5515ca 100755 --- a/deploy/convert_server.py +++ b/deploy/convert_server.py @@ -7,8 +7,10 @@ import json, re, os # ── DB auto-seed ────────────────────────────────────────────────────────── from db import prompts as db_prompts -from db.connection import DB_CONFIG +from db.connection import DB_CONFIG, execute db_prompts.seed_defaults() +# Schema migration: classify_raw column (idempotent) +execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_raw text") # ── DB modules ──────────────────────────────────────────────────────────── from db import supplements as db_supplements @@ -146,6 +148,13 @@ class Handler(BaseHTTPRequestHandler): "filename": doc["filename"], "status": doc["status"], "elements_json": doc.get("elements_json"), + "doc_type": doc.get("doc_type"), + "own_number": doc.get("own_number"), + "parent_number": doc.get("parent_number"), + "doc_date": doc.get("doc_date"), + "counterparty": doc.get("counterparty"), + "classify_status": doc.get("classify_status"), + "classify_raw": doc.get("classify_raw"), }) def _handle_api_document_delete(self, parsed): diff --git a/deploy/db/documents.py b/deploy/db/documents.py index e6ec1a6..5ede938 100644 --- a/deploy/db/documents.py +++ b/deploy/db/documents.py @@ -36,13 +36,13 @@ def delete(doc_id): return execute("DELETE FROM documents WHERE id = %s", (doc_id,)) -def set_classification(doc_id, doc_type, own_number, parent_number, doc_date, counterparty): - """Store LLM classification results.""" +def set_classification(doc_id, doc_type, own_number, parent_number, doc_date, counterparty, classify_raw=None): + """Store LLM classification results + raw response.""" return execute( """UPDATE documents SET doc_type=%s, own_number=%s, parent_number=%s, - doc_date=%s, counterparty=%s, classify_status='classified' + doc_date=%s, counterparty=%s, classify_status='classified', classify_raw=%s WHERE id=%s""", - (doc_type, own_number, parent_number, doc_date, counterparty, doc_id), + (doc_type, own_number, parent_number, doc_date, counterparty, classify_raw, doc_id), ) @@ -73,7 +73,7 @@ def list_by_batch(batch_id): """All documents in a batch with classification fields.""" return query( """SELECT id, filename, status, doc_type, own_number, parent_number, - doc_date, counterparty, classify_status, error_message + doc_date, counterparty, classify_status, error_message, classify_raw FROM documents WHERE batch_id=%s ORDER BY created_at""", (batch_id,), ) diff --git a/deploy/services/classify.py b/deploy/services/classify.py index 7064fa2..1bb5866 100644 --- a/deploy/services/classify.py +++ b/deploy/services/classify.py @@ -31,15 +31,14 @@ LLM_MODEL = "gpt-oss-120b" def _call_llm_classify(header_text): """ Прямой вызов LLM для классификации ОДНОГО документа. - Не использует общий call_llm() из services/llm.py — здесь свой промпт и формат ответа. - Возвращает распарсенный JSON dict с полями: doc_type, own_number, parent_number, doc_date, counterparty, confidence. + Возвращает (parsed_dict, raw_text). """ prompt, _ = build_classify_prompt(header_text) payload = { "model": LLM_MODEL, "messages": [{"role": "user", "content": prompt}], - "max_tokens": 1000, # достаточно для JSON с длинными названиями - "temperature": 0.1, # минимальная температура для детерминированности + "max_tokens": 1000, + "temperature": 0.1, } with httpx.Client(http2=True, timeout=60, verify=False) as client: resp = client.post( @@ -50,7 +49,7 @@ def _call_llm_classify(header_text): data = resp.json() raw = data.get("choices", [{}])[0].get("message", {}).get("content", "") - return _safe_json_parse(raw) + return _safe_json_parse(raw), raw def classify_batch(batch_id): @@ -74,7 +73,7 @@ def classify_batch(batch_id): """Классифицировать один документ: выжимка → LLM → сохранить результат.""" try: text = _smart_extract(doc["elements_json"]) - result = _call_llm_classify(text) + result, raw = _call_llm_classify(text) db_docs.set_classification( doc["id"], result.get("doc_type", "other"), @@ -82,6 +81,7 @@ def classify_batch(batch_id): result.get("parent_number"), result.get("doc_date"), result.get("counterparty"), + classify_raw=raw, ) return True except Exception as e: diff --git a/deploy/services/grouping.py b/deploy/services/grouping.py index 1f82af8..d3a4b0f 100644 --- a/deploy/services/grouping.py +++ b/deploy/services/grouping.py @@ -33,8 +33,9 @@ def group_documents(batch_id): 1. Отделить contract от supplement/specification 2. Каждый contract → якорь группы 3. Для каждого supplement: найти contract по parent_number (нормализованный) - 4. Несматченные → группа "__unresolved__" - 5. Внутри группы сортировка по doc_date + 4. Оставшиеся supplement/spec → виртуальные группы по parent_number/own_number + 5. Совсем без номеров → группа "__unresolved__" + 6. Внутри группы сортировка по doc_date Возвращает {ok, groups: [{contract_number, counterparty, documents: [...]}], total_docs}. """ @@ -79,18 +80,43 @@ def group_documents(batch_id): groups.append(group) - # Unmatched documents → "unresolved" group (including failed/pending) - unmatched = [s for s in supplements_list if s.get("doc_type") != "contract"] + # ── Virtual groups: unmatched supplements grouped by number ────────── + # Group remaining supplements/specs by normalized parent_number (priority) or own_number + virtual = {} + for s in supplements_list: + num = normalize_number(s.get("parent_number") or s.get("own_number") or "") + if not num: + continue # no number → stays in supplements_list for unresolved + if num not in virtual: + # Use counterparty from first doc in group + cp = s.get("counterparty") or "" + virtual[num] = { + "contract_number": s.get("parent_number") or s.get("own_number") or "?", + "counterparty": cp, + "documents": [], + } + virtual[num]["documents"].append(s) + # Update counterparty if current doc has a better one + if not virtual[num]["counterparty"] and s.get("counterparty"): + virtual[num]["counterparty"] = s.get("counterparty") + + for vnum, vgroup in virtual.items(): + vgroup["documents"].sort(key=lambda x: x.get("doc_date") or "") + groups.append(vgroup) + # Remove grouped docs from supplements_list + for s in vgroup["documents"]: + supplements_list.remove(s) + + # ── Unresolved: everything left (no number, failed, pending, other) ── + unmatched = [s for s in supplements_list] # remaining after virtual grouping unmatched += [d for d in docs if d.get("classify_status") != "classified"] - if unmatched or len(contracts_list) == 0: - # Put unmatched into a separate group - if unmatched: - groups.append({ - "contract_number": "__unresolved__", - "counterparty": "", - "documents": unmatched, - }) + if unmatched: + groups.append({ + "contract_number": "__unresolved__", + "counterparty": "", + "documents": unmatched, + }) return {"ok": True, "groups": groups, "total_docs": len(docs)} diff --git a/index.cfm b/index.cfm index f057388..3cc0d4a 100644 --- a/index.cfm +++ b/index.cfm @@ -75,7 +75,13 @@
Nubes - Сверка договоров — LLM AI-driven Event Sourcing v1.0.174 + Сверка договоров — LLM AI-driven Event Sourcing v1.0.175 +
+ ○ Загрузка→ + ○ Классификация→ + ○ Группировка→ + ○ Сравнение +
ℹ️ О сервисе