Compare commits
5
Commits
193f0b4069
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
79de42da96 | ||
|
|
9c73a730d1 | ||
|
|
05205b09cc | ||
|
|
2531a7c3c1 | ||
|
|
e2ba9ab451 |
+1
-1
@@ -1,7 +1,7 @@
|
||||
"""Конфигурация приложения — все настройки в одном месте."""
|
||||
import os
|
||||
|
||||
VERSION = "3.0.0"
|
||||
VERSION = "3.0.3"
|
||||
|
||||
|
||||
def _load_env():
|
||||
|
||||
+18
-2
@@ -12,7 +12,7 @@ import time
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
|
||||
DB_PATH = "/tmp/contracts.db"
|
||||
DB_PATH = os.getenv("CONTRACTS_DB_PATH", "/tmp/contracts.db")
|
||||
_db_session_key = None
|
||||
_local = threading.local()
|
||||
|
||||
@@ -67,6 +67,19 @@ def _create_schema(conn):
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS extraction_jobs (
|
||||
batch_id TEXT PRIMARY KEY,
|
||||
owner TEXT NOT NULL,
|
||||
state TEXT NOT NULL,
|
||||
total INTEGER NOT NULL,
|
||||
done INTEGER NOT NULL DEFAULT 0,
|
||||
failed INTEGER NOT NULL DEFAULT 0,
|
||||
classes_json TEXT NOT NULL DEFAULT '{}',
|
||||
result_json TEXT,
|
||||
error TEXT,
|
||||
updated_at REAL NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS contracts (
|
||||
id TEXT PRIMARY KEY,
|
||||
number TEXT NOT NULL,
|
||||
@@ -332,7 +345,10 @@ def execute_returning(sql, params=None):
|
||||
sql = sql[:sql.upper().rfind(" RETURNING ")]
|
||||
|
||||
cur = conn.execute(sql, params or [])
|
||||
conn.commit()
|
||||
# The outer transaction owns commit and rollback; committing here would make
|
||||
# its rollback ineffective and could expose only part of a multi-write update.
|
||||
if not getattr(_local, "in_transaction", False):
|
||||
conn.commit()
|
||||
rowid = cur.lastrowid
|
||||
|
||||
if table and rowid:
|
||||
|
||||
+2
-3
@@ -7,7 +7,7 @@
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from db.connection import execute, query, transaction
|
||||
from db.connection import execute, query
|
||||
|
||||
|
||||
def _json_value(value, default):
|
||||
@@ -123,5 +123,4 @@ def list_dossier_documents(dossier_id):
|
||||
|
||||
def clear_batch_dossiers(batch_id):
|
||||
"""Удалить все досье batch; таблицы с ON DELETE CASCADE очистятся вместе с ними."""
|
||||
with transaction():
|
||||
return execute("DELETE FROM contract_dossiers WHERE batch_id = %s", (batch_id,))
|
||||
return execute("DELETE FROM contract_dossiers WHERE batch_id = %s", (batch_id,))
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Persistent extraction progress and per-batch worker ownership."""
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from db.connection import execute, query
|
||||
|
||||
LEASE_SECONDS = 300
|
||||
|
||||
|
||||
def claim(batch_id, total):
|
||||
"""Claim a batch atomically, allowing retry after failure or expired lease."""
|
||||
owner = str(uuid.uuid4())
|
||||
now = time.time()
|
||||
changed = execute(
|
||||
"""INSERT INTO extraction_jobs (batch_id, owner, state, total, updated_at)
|
||||
VALUES (%s, %s, 'running', %s, %s)
|
||||
ON CONFLICT(batch_id) DO UPDATE SET owner=excluded.owner, state='running',
|
||||
total=excluded.total, done=0, failed=0, classes_json='{}',
|
||||
result_json=NULL, error=NULL, updated_at=excluded.updated_at
|
||||
WHERE extraction_jobs.state != 'running' OR extraction_jobs.updated_at < %s""",
|
||||
(batch_id, owner, total, now, now - LEASE_SECONDS),
|
||||
)
|
||||
return owner if changed else None
|
||||
|
||||
|
||||
def heartbeat(batch_id, owner):
|
||||
return execute(
|
||||
"UPDATE extraction_jobs SET updated_at=%s WHERE batch_id=%s AND owner=%s AND state='running'",
|
||||
(time.time(), batch_id, owner),
|
||||
)
|
||||
|
||||
|
||||
def progress(batch_id, owner, completed, result):
|
||||
rows = query(
|
||||
"SELECT failed, classes_json FROM extraction_jobs WHERE batch_id=%s AND owner=%s AND state='running'",
|
||||
(batch_id, owner),
|
||||
)
|
||||
if not rows:
|
||||
return
|
||||
failed = rows[0]["failed"]
|
||||
classes = json.loads(rows[0]["classes_json"])
|
||||
if not result.get("ok"):
|
||||
failed += 1
|
||||
else:
|
||||
doc_class = (result.get("extraction") or {}).get("doc_class", "unknown")
|
||||
classes[doc_class] = classes.get(doc_class, 0) + 1
|
||||
execute(
|
||||
"""UPDATE extraction_jobs SET done=%s, failed=%s, classes_json=%s, updated_at=%s
|
||||
WHERE batch_id=%s AND owner=%s AND state='running'""",
|
||||
(completed, failed, json.dumps(classes), time.time(), batch_id, owner),
|
||||
)
|
||||
|
||||
|
||||
def finish(batch_id, owner, result=None, error=None):
|
||||
state = "done" if result and result.get("ok") and not error else "failed"
|
||||
return execute(
|
||||
"""UPDATE extraction_jobs SET state=%s, result_json=%s, error=%s, updated_at=%s
|
||||
WHERE batch_id=%s AND owner=%s AND state='running'""",
|
||||
(state, json.dumps(result, ensure_ascii=False) if result is not None else None,
|
||||
error or ("extraction failed" if state == "failed" else None), time.time(), batch_id, owner),
|
||||
)
|
||||
|
||||
|
||||
def get(batch_id):
|
||||
"""Mark an abandoned lease interrupted; callers may explicitly retry it."""
|
||||
execute(
|
||||
"""UPDATE extraction_jobs SET state='interrupted', error='worker stopped; retry the batch'
|
||||
WHERE batch_id=%s AND state='running' AND updated_at < %s""",
|
||||
(batch_id, time.time() - LEASE_SECONDS),
|
||||
)
|
||||
rows = query("SELECT * FROM extraction_jobs WHERE batch_id=%s", (batch_id,))
|
||||
if not rows:
|
||||
return None
|
||||
job = rows[0]
|
||||
job["classes"] = json.loads(job.pop("classes_json"))
|
||||
result_json = job.pop("result_json")
|
||||
job["result"] = json.loads(result_json) if result_json else None
|
||||
return job
|
||||
+209
-70
@@ -1,18 +1,26 @@
|
||||
"""Pipeline blueprint — SSE-сравнение + classify."""
|
||||
import json, re, os, threading
|
||||
import hashlib, hmac, json, re, os, threading, time
|
||||
from flask import Blueprint, request, jsonify, Response, stream_with_context
|
||||
from config import API_KEY
|
||||
from services.process import run_pipeline
|
||||
from services import extractor as stateless_extractor
|
||||
from services.engine import analyze_batch
|
||||
from services.grouping import build_dossiers_for_batch
|
||||
from services.timeline_builder import build_dossier_timeline
|
||||
from llm_prompt import build_prompt
|
||||
from db import documents
|
||||
from db import documents, extraction_jobs
|
||||
from db.connection import transaction
|
||||
|
||||
pipeline_bp = Blueprint("pipeline", __name__)
|
||||
|
||||
# In-memory lock для classify (замена файлового lock)
|
||||
_classify_locks: dict[str, threading.Thread] = {}
|
||||
_extraction_jobs: dict[str, dict] = {}
|
||||
_engine_rate_lock = threading.Lock()
|
||||
_engine_rate_hits: dict[str, list[float]] = {}
|
||||
_ENGINE_MAX_BODY_BYTES = 10 * 1024 * 1024
|
||||
_ENGINE_MAX_DOCUMENTS = 100
|
||||
_ENGINE_MAX_ELEMENTS = 20_000
|
||||
_ENGINE_MAX_DOCUMENT_BYTES = 2 * 1024 * 1024
|
||||
_ENGINE_RATE_WINDOW_SECONDS = 60
|
||||
_ENGINE_RATE_LIMIT = 10
|
||||
|
||||
|
||||
@pipeline_bp.route("/process-v2", methods=["GET"])
|
||||
@@ -110,85 +118,214 @@ def classify_batch_route():
|
||||
if not isinstance(batch_id, str) or not batch_id.strip():
|
||||
return jsonify(ok=False, error="batch_id required"), 400
|
||||
|
||||
# Guard: уже запущена?
|
||||
if batch_id in _classify_locks:
|
||||
t = _classify_locks[batch_id]
|
||||
if t.is_alive():
|
||||
return jsonify(ok=False, error="classify already running"), 409
|
||||
else:
|
||||
del _classify_locks[batch_id]
|
||||
|
||||
pending = documents.list_for_extraction(batch_id)
|
||||
total = len(pending)
|
||||
if total == 0:
|
||||
return jsonify(ok=False, error="no parsed documents"), 400
|
||||
owner = extraction_jobs.claim(batch_id, total)
|
||||
if not owner:
|
||||
return jsonify(ok=False, error="classify already running"), 409
|
||||
|
||||
# Keep small batches synchronous for the existing browser workflow.
|
||||
if total <= 10:
|
||||
result = _run_v3_extraction(batch_id, total)
|
||||
result = _run_extraction_job(batch_id, pending, owner)
|
||||
return jsonify(result)
|
||||
|
||||
_extraction_jobs[batch_id] = {
|
||||
"state": "running", "total": total, "done": 0,
|
||||
"failed": 0, "classes": {},
|
||||
}
|
||||
|
||||
def _run():
|
||||
try:
|
||||
_run_v3_extraction(batch_id, total, _extraction_progress)
|
||||
_extraction_jobs[batch_id]["state"] = "done"
|
||||
except Exception as error:
|
||||
_extraction_jobs[batch_id].update(state="failed", error=str(error))
|
||||
finally:
|
||||
_classify_locks.pop(batch_id, None)
|
||||
|
||||
t = threading.Thread(target=_run, daemon=True)
|
||||
_classify_locks[batch_id] = t
|
||||
t.start()
|
||||
|
||||
_start_extraction_job(batch_id, pending, owner)
|
||||
return jsonify(ok=True, total=total, v3=True), 202
|
||||
|
||||
|
||||
def _extraction_progress(batch_id, completed, result):
|
||||
"""Обновить лёгкий in-memory прогресс для крупного batch."""
|
||||
job = _extraction_jobs[batch_id]
|
||||
job["done"] = completed
|
||||
if not result.get("ok"):
|
||||
job["failed"] += 1
|
||||
else:
|
||||
extraction = result.get("extraction") or {}
|
||||
doc_class = extraction.get("doc_class", "unknown")
|
||||
job["classes"][doc_class] = job["classes"].get(doc_class, 0) + 1
|
||||
@pipeline_bp.route("/api/engine/analyze", methods=["POST"])
|
||||
def engine_analyze_route():
|
||||
"""Accept parsed documents over a versioned, authenticated JSON boundary.
|
||||
|
||||
Validation is completed for the whole payload before opening the write
|
||||
transaction, so malformed input cannot leave a partial batch. The caller
|
||||
supplies parsed elements only; extraction passports and dossier decisions
|
||||
are always produced by the in-process Engine and persisted by its adapters.
|
||||
"""
|
||||
if not API_KEY:
|
||||
return jsonify(ok=False, error="engine API is not configured"), 503
|
||||
supplied_key = request.headers.get("X-API-Key", "")
|
||||
if not hmac.compare_digest(supplied_key.encode("utf-8"), API_KEY.encode("utf-8")):
|
||||
return jsonify(ok=False, error="unauthorized"), 401
|
||||
if _engine_rate_limited():
|
||||
return jsonify(ok=False, error="rate limit exceeded"), 429, {"Retry-After": "60"}
|
||||
|
||||
raw_body = request.stream.read(_ENGINE_MAX_BODY_BYTES + 1)
|
||||
if len(raw_body) > _ENGINE_MAX_BODY_BYTES:
|
||||
return jsonify(ok=False, error="request body too large"), 413
|
||||
try:
|
||||
body = json.loads(raw_body) if request.is_json else None
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
body = None
|
||||
if not isinstance(body, dict):
|
||||
return jsonify(ok=False, error="JSON object required"), 400
|
||||
if type(body.get("schema_version")) is not int or body["schema_version"] != 1:
|
||||
return jsonify(ok=False, error="unsupported schema_version"), 400
|
||||
|
||||
batch_id = body.get("batch_id")
|
||||
if not isinstance(batch_id, str) or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", batch_id):
|
||||
return jsonify(ok=False, error="invalid batch_id"), 400
|
||||
source_documents = body.get("documents")
|
||||
if not isinstance(source_documents, list) or not source_documents:
|
||||
return jsonify(ok=False, error="documents must be a non-empty array"), 400
|
||||
if len(source_documents) > _ENGINE_MAX_DOCUMENTS:
|
||||
return jsonify(ok=False, error="too many documents"), 413
|
||||
validated = []
|
||||
for index, source_document in enumerate(source_documents):
|
||||
if not isinstance(source_document, dict):
|
||||
return jsonify(ok=False, error=f"documents[{index}] must be an object"), 400
|
||||
filename = source_document.get("filename")
|
||||
elements = source_document.get("elements")
|
||||
if not isinstance(filename, str) or not filename.strip() or len(filename) > 512:
|
||||
return jsonify(ok=False, error=f"documents[{index}].filename is invalid"), 400
|
||||
if not isinstance(elements, list) or len(elements) > _ENGINE_MAX_ELEMENTS:
|
||||
return jsonify(ok=False, error=f"documents[{index}].elements is invalid"), 400
|
||||
for element_index, element in enumerate(elements):
|
||||
if not _valid_engine_element(element):
|
||||
return jsonify(
|
||||
ok=False, error=f"documents[{index}].elements[{element_index}] is invalid",
|
||||
), 400
|
||||
try:
|
||||
elements_json = json.dumps(
|
||||
elements, ensure_ascii=False, sort_keys=True, separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return jsonify(ok=False, error=f"documents[{index}].elements is not valid JSON"), 400
|
||||
encoded_elements = elements_json.encode("utf-8")
|
||||
if len(encoded_elements) > _ENGINE_MAX_DOCUMENT_BYTES:
|
||||
return jsonify(ok=False, error=f"documents[{index}] is too large"), 413
|
||||
validated.append((filename.strip(), elements, elements_json, encoded_elements))
|
||||
|
||||
inserted = []
|
||||
with transaction():
|
||||
# Engine API owns a fresh batch per request. Preventing reuse inside
|
||||
# BEGIN IMMEDIATE keeps old parsed rows out of this run and closes the
|
||||
# race between two concurrent requests using the same batch ID.
|
||||
if documents.list_by_batch(batch_id):
|
||||
return jsonify(ok=False, error="batch_id already exists"), 409
|
||||
owner = extraction_jobs.claim(batch_id, len(validated))
|
||||
if not owner:
|
||||
return jsonify(ok=False, error="classify already running"), 409
|
||||
for filename, elements, elements_json, encoded_elements in validated:
|
||||
content_hash = hashlib.sha256(encoded_elements).hexdigest()
|
||||
document = documents.insert(
|
||||
filename, "application/json", None, status="uploaded",
|
||||
batch_id=batch_id, content_hash=content_hash,
|
||||
)
|
||||
documents.set_parsed(document["id"], elements)
|
||||
inserted.append({"id": document["id"], "filename": filename})
|
||||
|
||||
pending = documents.list_for_extraction(batch_id)
|
||||
total = len(pending)
|
||||
if total > 10:
|
||||
_start_extraction_job(batch_id, pending, owner)
|
||||
return jsonify(ok=True, v3=True, total=total, documents=inserted), 202
|
||||
|
||||
result = _run_extraction_job(batch_id, pending, owner)
|
||||
result["documents"] = inserted
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
def _run_v3_extraction(batch_id, total, progress_callback=None):
|
||||
def _valid_engine_element(element):
|
||||
"""Validate the parser fields consumed by structured-text formatting."""
|
||||
if not isinstance(element, dict):
|
||||
return False
|
||||
if element.get("type") == "paragraph":
|
||||
return isinstance(element.get("text"), str) and isinstance(element.get("style", ""), str)
|
||||
if element.get("type") != "table":
|
||||
return False
|
||||
rows = element.get("rows")
|
||||
context = element.get("preceding_paragraphs", [])
|
||||
table_index = element.get("table_index")
|
||||
return (
|
||||
isinstance(rows, list)
|
||||
and all(isinstance(row, list) and all(isinstance(cell, str) for cell in row) for row in rows)
|
||||
and isinstance(context, list)
|
||||
and all(isinstance(paragraph, str) for paragraph in context)
|
||||
and (table_index is None or (type(table_index) is int and table_index > 0))
|
||||
)
|
||||
|
||||
|
||||
def _start_extraction_job(batch_id, pending, owner):
|
||||
"""Run a claimed batch in the background with persistent progress."""
|
||||
def _run():
|
||||
try:
|
||||
_run_extraction_job(batch_id, pending, owner)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
thread = threading.Thread(target=_run, daemon=True)
|
||||
try:
|
||||
thread.start()
|
||||
except Exception as error:
|
||||
extraction_jobs.finish(batch_id, owner, error=str(error))
|
||||
raise
|
||||
|
||||
|
||||
def _engine_rate_limited():
|
||||
"""Apply a small per-source fixed-window limit before accepting LLM work."""
|
||||
now = time.monotonic()
|
||||
source = request.remote_addr or "unknown"
|
||||
cutoff = now - _ENGINE_RATE_WINDOW_SECONDS
|
||||
with _engine_rate_lock:
|
||||
for key in list(_engine_rate_hits):
|
||||
recent = [stamp for stamp in _engine_rate_hits[key] if stamp > cutoff]
|
||||
if recent:
|
||||
_engine_rate_hits[key] = recent
|
||||
else:
|
||||
del _engine_rate_hits[key]
|
||||
recent = _engine_rate_hits.get(source, [])
|
||||
if len(recent) >= _ENGINE_RATE_LIMIT:
|
||||
return True
|
||||
recent.append(now)
|
||||
_engine_rate_hits[source] = recent
|
||||
return False
|
||||
|
||||
|
||||
def _run_extraction_job(batch_id, pending, owner):
|
||||
"""Keep the lease alive during LLM calls and persist every final outcome."""
|
||||
stop = threading.Event()
|
||||
|
||||
def renew_lease():
|
||||
while not stop.wait(15):
|
||||
if not extraction_jobs.heartbeat(batch_id, owner):
|
||||
return
|
||||
|
||||
heartbeat = threading.Thread(target=renew_lease, daemon=True)
|
||||
heartbeat.start()
|
||||
try:
|
||||
result = _run_v3_extraction(
|
||||
batch_id, pending,
|
||||
progress_callback=lambda completed, item: extraction_jobs.progress(batch_id, owner, completed, item),
|
||||
)
|
||||
extraction_jobs.finish(batch_id, owner, result=result)
|
||||
return result
|
||||
except Exception as error:
|
||||
extraction_jobs.finish(batch_id, owner, error=str(error))
|
||||
raise
|
||||
finally:
|
||||
stop.set()
|
||||
heartbeat.join()
|
||||
|
||||
|
||||
def _run_v3_extraction(batch_id, pending, progress_callback=None, llm_client=None):
|
||||
"""Собрать extraction и dossier summaries в один ответ API."""
|
||||
extraction = stateless_extractor.extract_batch(
|
||||
parsed_documents = [{
|
||||
"id": document["id"],
|
||||
"filename": document["filename"],
|
||||
"elements": stateless_extractor._decode_elements(document.get("elements_json")),
|
||||
} for document in pending]
|
||||
return analyze_batch(
|
||||
batch_id,
|
||||
progress_callback=(
|
||||
(lambda completed, result: progress_callback(batch_id, completed, result))
|
||||
if progress_callback else None
|
||||
),
|
||||
parsed_documents,
|
||||
extractor=stateless_extractor.extract_batch,
|
||||
dossier_builder=build_dossiers_for_batch,
|
||||
llm_client=llm_client,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
dossier_result = build_dossiers_for_batch(batch_id)
|
||||
classes = extraction.get("classes", {})
|
||||
math_checks_passed = sum(
|
||||
1 for result in extraction.get("results", [])
|
||||
if (result.get("extraction") or {}).get("math_validation", {}).get("status") == "VERIFIED"
|
||||
)
|
||||
return {
|
||||
"ok": bool(extraction.get("results")) and dossier_result.get("ok", False),
|
||||
"v3": True,
|
||||
"done": total,
|
||||
"total": total,
|
||||
"failed": extraction.get("failed", 0),
|
||||
"classes": classes,
|
||||
"ignored_accounting": classes.get("ignored_accounting", 0),
|
||||
"math_checks_passed": math_checks_passed,
|
||||
"dossier_count": len(dossier_result.get("dossiers", [])),
|
||||
"ambiguous": dossier_result.get("ambiguous", 0),
|
||||
"unlinked": dossier_result.get("unlinked", []),
|
||||
}
|
||||
|
||||
|
||||
@pipeline_bp.route("/api/classify-progress")
|
||||
@@ -197,13 +334,13 @@ def classify_progress_route():
|
||||
batch_id = request.args.get("batch")
|
||||
if not batch_id:
|
||||
return jsonify(ok=False, error="batch required"), 400
|
||||
job = _extraction_jobs.get(batch_id)
|
||||
job = extraction_jobs.get(batch_id)
|
||||
if not job:
|
||||
return jsonify(ok=False, error="batch extraction not found"), 404
|
||||
response = {
|
||||
"ok": job["state"] != "failed",
|
||||
"ok": job["state"] not in {"failed", "interrupted"},
|
||||
"v3": True,
|
||||
"finished": job["state"] in {"done", "failed"},
|
||||
"finished": job["state"] in {"done", "failed", "interrupted"},
|
||||
"state": job["state"],
|
||||
"done": job["done"],
|
||||
"total": job["total"],
|
||||
@@ -212,4 +349,6 @@ def classify_progress_route():
|
||||
}
|
||||
if job.get("error"):
|
||||
response["error"] = job["error"]
|
||||
if job.get("result") is not None:
|
||||
response["result"] = job["result"]
|
||||
return jsonify(response)
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Shared orchestration for the contract reconstruction engine.
|
||||
|
||||
This module deliberately accepts the batch's persistence-backed extractor and
|
||||
dossier builder as dependencies. Both the browser workflow and the JSON
|
||||
integration boundary therefore execute the same business flow in-process,
|
||||
while tests can replace either dependency without constructing HTTP calls or
|
||||
making network requests.
|
||||
"""
|
||||
|
||||
|
||||
def analyze_batch(batch_id, documents, extractor, dossier_builder,
|
||||
progress_callback=None, llm_client=None):
|
||||
"""Extract document passports, assemble dossiers, and shape the API result.
|
||||
|
||||
`extractor` owns LLM interaction and persistence of each extraction;
|
||||
`dossier_builder` consumes those saved passports and persists the resulting
|
||||
graph. Keeping the calls in this order is an invariant: grouping must never
|
||||
observe a partially extracted batch. The callback remains optional because
|
||||
small batches run synchronously, while the existing large-batch route uses
|
||||
it to update persistent progress.
|
||||
"""
|
||||
extraction = extractor(
|
||||
batch_id,
|
||||
llm_client=llm_client,
|
||||
progress_callback=progress_callback,
|
||||
documents=documents,
|
||||
)
|
||||
# Extractor-level `ok` means every input succeeded; dossier construction only
|
||||
# requires at least one persisted passport and can proceed for that subset.
|
||||
extraction_results = extraction.get("results") or []
|
||||
total = extraction.get("total", len(documents))
|
||||
done = extraction.get("done")
|
||||
if done is None:
|
||||
done = sum(1 for result in extraction_results if result.get("ok"))
|
||||
failed = extraction.get("failed")
|
||||
if failed is None:
|
||||
failed = max(total - done, 0)
|
||||
dossier_result = dossier_builder(batch_id) if done > 0 else {"ok": False}
|
||||
dossier_ok = bool(dossier_result.get("ok"))
|
||||
# Keep result usability separate from completeness so clients can continue
|
||||
# with valid dossiers while still reporting failures explicitly.
|
||||
status = (
|
||||
"failed" if done == 0 or not dossier_ok
|
||||
else "partial" if failed > 0
|
||||
else "completed"
|
||||
)
|
||||
documents_by_id = {document.get("id"): document for document in documents}
|
||||
failed_documents = []
|
||||
for result in extraction_results:
|
||||
if result.get("ok"):
|
||||
continue
|
||||
document_id = result.get("document_id")
|
||||
source_document = documents_by_id.get(document_id, {})
|
||||
failure = {
|
||||
"document_id": document_id,
|
||||
"filename": source_document.get("filename"),
|
||||
"error": result.get("error") or "extraction failed",
|
||||
}
|
||||
if result.get("persistence_error"):
|
||||
failure["persistence_error"] = result["persistence_error"]
|
||||
failed_documents.append(failure)
|
||||
classes = extraction.get("classes", {})
|
||||
math_checks_passed = sum(
|
||||
1 for result in extraction_results
|
||||
if (result.get("extraction") or {}).get("math_validation", {}).get("status") == "VERIFIED"
|
||||
)
|
||||
return {
|
||||
"ok": status != "failed",
|
||||
"status": status,
|
||||
"partial": status == "partial",
|
||||
"v3": True,
|
||||
"done": done,
|
||||
"total": total,
|
||||
"failed": failed,
|
||||
"failed_documents": failed_documents,
|
||||
"classes": classes,
|
||||
"ignored_accounting": classes.get("ignored_accounting", 0),
|
||||
"math_checks_passed": math_checks_passed,
|
||||
"dossier_count": len(dossier_result.get("dossiers", [])),
|
||||
"ambiguous": dossier_result.get("ambiguous", 0),
|
||||
"unlinked": dossier_result.get("unlinked", []),
|
||||
}
|
||||
+24
-11
@@ -250,12 +250,16 @@ def _save_extraction(document_id, extracted, validation, status, raw_response="{
|
||||
|
||||
def extract_document(document, llm_client=None, batch_id=None):
|
||||
"""Извлечь паспорт одного документа, изолируя ошибки на уровне документа."""
|
||||
document_id = document["id"]
|
||||
batch_id = batch_id if batch_id is not None else document.get("batch_id")
|
||||
elements = _decode_elements(document.get("elements_json"))
|
||||
structured_text = format_elements_with_structure(elements)
|
||||
document_id = document.get("id") if isinstance(document, dict) else None
|
||||
raw_response = "{}"
|
||||
try:
|
||||
if not document_id:
|
||||
raise ValueError("document id is required")
|
||||
batch_id = batch_id if batch_id is not None else document.get("batch_id")
|
||||
elements = _decode_elements(document.get("elements", document.get("elements_json")))
|
||||
# Formatting is part of this document's failure boundary: corrupt stored
|
||||
# elements must not escape and stop extraction of later batch members.
|
||||
structured_text = format_elements_with_structure(elements)
|
||||
if _is_garbage_by_filename(document.get("filename") or "") or _is_garbage_by_header(structured_text[:2000]):
|
||||
extracted = {
|
||||
"doc_class": "ignored_accounting", "intent_mode": "no_spec",
|
||||
@@ -299,20 +303,29 @@ def extract_document(document, llm_client=None, batch_id=None):
|
||||
return {"ok": True, "document_id": document_id, "status": result["status"], "extraction": saved}
|
||||
except Exception as error:
|
||||
log.exception("Stateless extraction failed for document %s", document_id)
|
||||
_save_extraction(document_id, {
|
||||
"doc_class": "unknown", "intent_mode": "no_spec", "counterparty": {},
|
||||
"referenced_contracts": [], "sections": [],
|
||||
}, {"status": "FAILED", "error": str(error)}, "FAILED", raw_response)
|
||||
return {"ok": False, "document_id": document_id, "error": str(error)}
|
||||
result = {"ok": False, "document_id": document_id, "error": str(error)}
|
||||
if document_id:
|
||||
# Failure persistence is best-effort; a secondary write failure must
|
||||
# not escape this handler and abort processing of remaining documents.
|
||||
try:
|
||||
_save_extraction(document_id, {
|
||||
"doc_class": "unknown", "intent_mode": "no_spec", "counterparty": {},
|
||||
"referenced_contracts": [], "sections": [],
|
||||
}, {"status": "FAILED", "error": str(error)}, "FAILED", raw_response)
|
||||
except Exception as persistence_error:
|
||||
log.exception("Failed to persist extraction failure for document %s", document_id)
|
||||
result["persistence_error"] = str(persistence_error)
|
||||
return result
|
||||
|
||||
|
||||
def extract_batch(batch_id, llm_client=None, progress_callback=None):
|
||||
def extract_batch(batch_id, llm_client=None, progress_callback=None, documents=None):
|
||||
"""Обработать документы batch и вернуть сводку без batch-wide failure.
|
||||
|
||||
Optional progress callback получает (completed_count, document_result) после
|
||||
каждого документа; ошибки callback не должны останавливать обработку batch.
|
||||
"""
|
||||
documents = db_docs.list_for_extraction(batch_id)
|
||||
if documents is None:
|
||||
documents = db_docs.list_for_extraction(batch_id)
|
||||
results = []
|
||||
for completed, document in enumerate(documents, start=1):
|
||||
result = extract_document(document, llm_client=llm_client, batch_id=batch_id)
|
||||
|
||||
@@ -15,6 +15,7 @@ from db import supplements as db_supplements
|
||||
from db import dossiers as db_dossiers
|
||||
from db import exceptions as db_exceptions
|
||||
from db import extractions as db_extractions
|
||||
from db.connection import transaction
|
||||
|
||||
|
||||
_HOMOGLYPHS = str.maketrans({
|
||||
@@ -164,6 +165,12 @@ def _dossier_exception(batch_id, document_id, title, source_quote):
|
||||
|
||||
|
||||
def build_dossiers_for_batch(batch_id):
|
||||
"""Atomically rebuild the dossier graph, retaining the old graph on failure."""
|
||||
with transaction():
|
||||
return _build_dossiers_for_batch(batch_id)
|
||||
|
||||
|
||||
def _build_dossiers_for_batch(batch_id):
|
||||
"""Построить и сохранить dossier graph из паспортов документов одного batch.
|
||||
|
||||
Полные номера и ссылки образуют рёбра только при совместимом контрагенте.
|
||||
@@ -177,8 +184,10 @@ def build_dossiers_for_batch(batch_id):
|
||||
records = []
|
||||
for document in documents:
|
||||
passport = passport_by_id.get(document["id"])
|
||||
if not passport:
|
||||
passport = {"document_id": document["id"], "doc_class": "unknown"}
|
||||
# A missing or failed passport says extraction produced no usable evidence;
|
||||
# do not misreport that technical failure as a missing contract number.
|
||||
if not passport or passport.get("status") == "FAILED":
|
||||
continue
|
||||
if passport.get("doc_class") == "ignored_accounting":
|
||||
continue
|
||||
counterparty = {
|
||||
|
||||
+85
-80
@@ -14,7 +14,7 @@ var VM_UPLOAD_URL = 'https://contracts-upload.sinefo.ru/contracts-upload/';
|
||||
// Прогрев приложения при загрузке страницы.
|
||||
// ⚠️ Без прогрева первый POST может упасть с ERR_CONNECTION_RESET (MTU/Geneve)
|
||||
(async function init() {
|
||||
try { await fetch('/health'); } catch(e) { /* ignore */ }
|
||||
try { await fetch('/health'); } catch (e) { /* ignore */ }
|
||||
})();
|
||||
|
||||
/**
|
||||
@@ -31,15 +31,15 @@ filePicker = window.FilePicker.initFilePicker({
|
||||
allowedExt: ['.pdf', '.doc', '.docx'],
|
||||
limits: FILE_PICKER_LIMITS,
|
||||
onChange: onPickerFilesChange,
|
||||
onError: function(error, file) {
|
||||
onError: function (error, file) {
|
||||
filePicker.setMessage('Ошибка чтения ' + file.name + ': ' + error.message);
|
||||
},
|
||||
onDetails: function(id) {
|
||||
var index = state.files.findIndex(function(file) { return file.pickerId === id; });
|
||||
onDetails: function (id) {
|
||||
var index = state.files.findIndex(function (file) { return file.pickerId === id; });
|
||||
if (index >= 0) toggleClassifyDetail(index);
|
||||
},
|
||||
onText: function(id) {
|
||||
var index = state.files.findIndex(function(file) { return file.pickerId === id; });
|
||||
onText: function (id) {
|
||||
var index = state.files.findIndex(function (file) { return file.pickerId === id; });
|
||||
if (index >= 0) showText(index);
|
||||
}
|
||||
});
|
||||
@@ -59,10 +59,10 @@ function renderStepper(state) {
|
||||
var steps = state.ui.steps;
|
||||
var map = { upload: 'stepUpload', classify: 'stepClassify', groups: 'stepGroups', compare: 'stepCompare' };
|
||||
var colors = { '○': 'var(--topbar-muted)', '⏳': 'var(--topbar-active)', '✓': 'var(--topbar-done)' };
|
||||
Object.keys(map).forEach(function(key) {
|
||||
Object.keys(map).forEach(function (key) {
|
||||
var el = document.getElementById(map[key]);
|
||||
if (el) {
|
||||
el.textContent = steps[key] + ' ' + {upload:'Загрузка',classify:'Классификация',groups:'Группировка',compare:'Сравнение'}[key];
|
||||
el.textContent = steps[key] + ' ' + { upload: 'Загрузка', classify: 'Классификация', groups: 'Группировка', compare: 'Сравнение' }[key];
|
||||
el.style.color = colors[steps[key]] || 'var(--muted)';
|
||||
}
|
||||
});
|
||||
@@ -92,10 +92,10 @@ function stepActive(id) {
|
||||
* Мутирует state.ui.steps, вызывает renderStepper.
|
||||
*/
|
||||
function resetStepper(fromId) {
|
||||
var order = ['stepUpload','stepClassify','stepGroups','stepCompare'];
|
||||
var keys = ['upload','classify','groups','compare'];
|
||||
var order = ['stepUpload', 'stepClassify', 'stepGroups', 'stepCompare'];
|
||||
var keys = ['upload', 'classify', 'groups', 'compare'];
|
||||
var reset = false;
|
||||
order.forEach(function(id, i) {
|
||||
order.forEach(function (id, i) {
|
||||
if (id === fromId) reset = true;
|
||||
if (reset) { state.ui.steps[keys[i]] = '○'; }
|
||||
});
|
||||
@@ -133,7 +133,7 @@ async function runClassify() {
|
||||
stepActive('stepClassify');
|
||||
|
||||
var start = Date.now();
|
||||
var timer = setInterval(function() {
|
||||
var timer = setInterval(function () {
|
||||
btn.innerHTML = '<span class="spinner"></span> Классификация... ' + Math.round((Date.now() - start) / 1000) + 'с';
|
||||
}, 1000);
|
||||
|
||||
@@ -141,7 +141,7 @@ async function runClassify() {
|
||||
var classifyDone = false;
|
||||
|
||||
// Poll progress каждые 2с — работает и для sync, и для async classify
|
||||
var pollTimer = setInterval(async function() {
|
||||
var pollTimer = setInterval(async function () {
|
||||
try {
|
||||
var progressUrl = state.v3Mode
|
||||
? '/api/classify-progress?batch=' + encodeURIComponent(state.batchId)
|
||||
@@ -157,12 +157,15 @@ async function runClassify() {
|
||||
clearInterval(pollTimer);
|
||||
clearInterval(timer);
|
||||
classifyDone = true;
|
||||
if (d.state === 'failed') {
|
||||
if (d.state === 'failed' || d.state === 'interrupted') {
|
||||
btn.innerHTML = '✗ ' + escHtml(d.error || 'ошибка извлечения');
|
||||
btn.disabled = false;
|
||||
return;
|
||||
}
|
||||
btn.innerHTML = '✓ ' + completed + '/' + total + ' документов извлечено';
|
||||
// Partial jobs are terminal and usable, but must not look like full success.
|
||||
var failed = d.failed || 0;
|
||||
btn.innerHTML = (failed ? '⚠ ' : '✓ ') + completed + '/' + total + ' документов извлечено'
|
||||
+ (failed ? ' | ошибок: ' + failed : '');
|
||||
stepDone('stepClassify');
|
||||
stepActive('stepGroups');
|
||||
loadGroupsAction();
|
||||
@@ -171,8 +174,8 @@ async function runClassify() {
|
||||
}
|
||||
if (d.ok && d.counts) {
|
||||
var done = (d.counts.classified || 0)
|
||||
+ (d.counts.failed || 0)
|
||||
+ (d.counts.garbage || 0);
|
||||
+ (d.counts.failed || 0)
|
||||
+ (d.counts.garbage || 0);
|
||||
var total = d.total || 0;
|
||||
if (total > 0) totalFiles = total;
|
||||
btn.innerHTML = '<span class="spinner"></span> Классификация... ' + done + '/' + total + ' (' + Math.round((Date.now() - start) / 1000) + 'с)';
|
||||
@@ -186,14 +189,14 @@ async function runClassify() {
|
||||
loadGroupsAction();
|
||||
}
|
||||
}
|
||||
} catch(e) {}
|
||||
} catch (e) { }
|
||||
}, 2000);
|
||||
|
||||
try {
|
||||
var resp = await fetch(VM_API + '/api/classify-batch', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({batch_id: state.batchId})
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ batch_id: state.batchId })
|
||||
});
|
||||
var data = await resp.json();
|
||||
state.v3Mode = Boolean(data.v3);
|
||||
@@ -211,8 +214,10 @@ async function runClassify() {
|
||||
if (data.v3) summary += ' | арифметика: ' + data.math_checks_passed;
|
||||
if (data.v3 && data.failed) summary += ' | ошибок: ' + data.failed;
|
||||
if (data.garbage) summary += ' | 🗑️ ' + data.garbage + ' мусор';
|
||||
if (data.types) summary += ' | ' + Object.entries(data.types).map(function(e){return e[1]+'×'+e[0];}).join(', ');
|
||||
btn.innerHTML = '✓ ' + summary + ' (' + Math.round((Date.now() - start) / 1000) + 'с)';
|
||||
if (data.types) summary += ' | ' + Object.entries(data.types).map(function (e) { return e[1] + '×' + e[0]; }).join(', ');
|
||||
// Continue to dossier rendering for partial success while keeping its warning visible.
|
||||
btn.innerHTML = (data.v3 && data.partial ? '⚠ ' : '✓ ') + summary
|
||||
+ ' (' + Math.round((Date.now() - start) / 1000) + 'с)';
|
||||
stepDone('stepClassify');
|
||||
stepActive('stepGroups');
|
||||
loadGroupsAction();
|
||||
@@ -231,7 +236,7 @@ async function runClassify() {
|
||||
btn.innerHTML = '✗ ' + (data.error || 'ошибка');
|
||||
btn.disabled = false;
|
||||
}
|
||||
} catch(e) {
|
||||
} catch (e) {
|
||||
if (!classifyDone) {
|
||||
clearInterval(timer);
|
||||
clearInterval(pollTimer);
|
||||
@@ -246,7 +251,7 @@ async function runClassify() {
|
||||
// Текущий активный EventSource (только один одновременно)
|
||||
// state._activeCompare — теперь в state.js (Фаза 0: state + render)
|
||||
|
||||
window.runCompareForGroup = async function(gi, btn) {
|
||||
window.runCompareForGroup = async function (gi, btn) {
|
||||
var group = state.groups[gi];
|
||||
if (!group || group.contract_number === '__unresolved__') return;
|
||||
|
||||
@@ -265,17 +270,17 @@ window.runCompareForGroup = async function(gi, btn) {
|
||||
btn.disabled = true;
|
||||
btn.innerHTML = '<span class="spinner"></span> Применяю...';
|
||||
// Заблокировать ВСЕ кнопки сравнения пока идёт процесс
|
||||
document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = true; });
|
||||
document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = true; });
|
||||
|
||||
var ar = await fetch(VM_API + '/api/apply-groups', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({batch_id: state.batchId, groups: [group]})
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ batch_id: state.batchId, groups: [group] })
|
||||
});
|
||||
var ad = await ar.json();
|
||||
if (!ad.ok || !ad.contract_ids || !ad.contract_ids.length) {
|
||||
btn.innerHTML = '✗ ошибка'; btn.disabled = false;
|
||||
document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = false; });
|
||||
document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = false; });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -306,7 +311,7 @@ window.runCompareForGroup = async function(gi, btn) {
|
||||
cmpBody,
|
||||
statusEl,
|
||||
{
|
||||
onDone: function(d, sections) {
|
||||
onDone: function (d, sections) {
|
||||
state._activeCompare.timer = null;
|
||||
state._activeCompare.es = null;
|
||||
group.compare = {
|
||||
@@ -316,23 +321,23 @@ window.runCompareForGroup = async function(gi, btn) {
|
||||
sections: sections
|
||||
};
|
||||
renderGroups(state);
|
||||
document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = false; });
|
||||
document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = false; });
|
||||
stepDone('stepCompare');
|
||||
},
|
||||
onError: function(d) {
|
||||
onError: function (d) {
|
||||
state._activeCompare.timer = null;
|
||||
state._activeCompare.es = null;
|
||||
group.compare = null;
|
||||
cmpBody.innerHTML += '<div style="color:var(--destructive);">✗ ' + escHtml(d.message) + '</div>';
|
||||
btn.innerHTML = '✗'; btn.disabled = false;
|
||||
document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = false; });
|
||||
document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = false; });
|
||||
},
|
||||
onConnectionError: function() {
|
||||
onConnectionError: function () {
|
||||
state._activeCompare.timer = null;
|
||||
state._activeCompare.es = null;
|
||||
group.compare = null;
|
||||
btn.innerHTML = '✗ соединение'; btn.disabled = false;
|
||||
document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = false; });
|
||||
document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = false; });
|
||||
}
|
||||
}
|
||||
);
|
||||
@@ -343,7 +348,7 @@ window.runCompareForGroup = async function(gi, btn) {
|
||||
// uploadFile → files.js (Фаза 1)
|
||||
|
||||
// ── LLM: Общее сравнение (Event Sourcing) ────────────────────────
|
||||
document.getElementById('llmBtn').addEventListener('click', function() {
|
||||
document.getElementById('llmBtn').addEventListener('click', function () {
|
||||
if (!state.contractId) return;
|
||||
var btn = this;
|
||||
btn.disabled = true;
|
||||
@@ -356,13 +361,13 @@ document.getElementById('llmBtn').addEventListener('click', function() {
|
||||
compareBody.innerHTML = '';
|
||||
compareStatus.textContent = '⏳ 0.0с';
|
||||
|
||||
var order = state.files.map(function(f) { return f.supp_id; }).filter(Boolean).join(',');
|
||||
var order = state.files.map(function (f) { return f.supp_id; }).filter(Boolean).join(',');
|
||||
var url = '/process-v2?contract_id=' + state.contractId +
|
||||
(order ? '&order=' + encodeURIComponent(order) : '');
|
||||
|
||||
// Фаза 3: унифицированный SSE через startCompareSSE (compare.js)
|
||||
startCompareSSE(url, compareBody, compareStatus, {
|
||||
onDone: function(d) {
|
||||
onDone: function (d) {
|
||||
btn.innerHTML = '<i data-lucide="check" style="width:16px;height:16px;"></i> ✓ Готово';
|
||||
stepDone('stepCompare');
|
||||
if (d.total_unresolved > 0) {
|
||||
@@ -371,12 +376,12 @@ document.getElementById('llmBtn').addEventListener('click', function() {
|
||||
document.getElementById('chatCard').style.display = 'block';
|
||||
lucide.createIcons();
|
||||
},
|
||||
onError: function(d) {
|
||||
onError: function (d) {
|
||||
compareBody.innerHTML += '<div style="color:var(--destructive);margin-top:8px;">✗ ' + escHtml(d.message) + '</div>';
|
||||
btn.innerHTML = '<i data-lucide="scale" style="width:16px;height:16px;"></i> Общее сравнение';
|
||||
btn.disabled = false;
|
||||
},
|
||||
onConnectionError: function() {
|
||||
onConnectionError: function () {
|
||||
if (compareBody.innerHTML === '') {
|
||||
compareBody.innerHTML = '<div style="color:var(--destructive);">✗ Ошибка соединения</div>';
|
||||
}
|
||||
@@ -390,7 +395,7 @@ document.getElementById('llmBtn').addEventListener('click', function() {
|
||||
// refreshSupps → files.js (Фаза 1)
|
||||
|
||||
// ── Чат ──────────────────────────────────────────────────────
|
||||
document.getElementById('chatSend').addEventListener('click', function() {
|
||||
document.getElementById('chatSend').addEventListener('click', function () {
|
||||
var input = document.getElementById('chatInput');
|
||||
var q = input.value.trim();
|
||||
if (!q || !state.contractId) return;
|
||||
@@ -404,21 +409,21 @@ document.getElementById('chatSend').addEventListener('click', function() {
|
||||
var fd = new FormData();
|
||||
fd.append('question', q);
|
||||
fetch('/chat.cfm?contract_id=' + state.contractId, { method: 'POST', body: fd })
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(d) {
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (d) {
|
||||
msgs.lastChild.remove();
|
||||
msgs.innerHTML += '<div style="margin-top:2px;"><strong>Ответ:</strong> ' + (d.OK ? d.ANSWER : (d.ERROR || '—')) + '</div>';
|
||||
input.disabled = false;
|
||||
document.getElementById('chatSend').disabled = false;
|
||||
input.focus();
|
||||
}).catch(function() {
|
||||
}).catch(function () {
|
||||
msgs.lastChild.remove();
|
||||
msgs.innerHTML += '<div style="color:var(--destructive);margin-top:2px;">Ошибка сети</div>';
|
||||
input.disabled = false;
|
||||
document.getElementById('chatSend').disabled = false;
|
||||
});
|
||||
});
|
||||
document.getElementById('chatInput').addEventListener('keydown', function(e) {
|
||||
document.getElementById('chatInput').addEventListener('keydown', function (e) {
|
||||
if (e.key === 'Enter') document.getElementById('chatSend').click();
|
||||
});
|
||||
|
||||
@@ -454,24 +459,24 @@ async function showText(i) {
|
||||
|
||||
// Общий textify для обеих панелей
|
||||
var textLines = [];
|
||||
elements.forEach(function(el) {
|
||||
elements.forEach(function (el) {
|
||||
var etype = el.type || el.TYPE || '?';
|
||||
if (etype === 'paragraph') {
|
||||
textLines.push(el.text || el.TEXT || '');
|
||||
} else if (etype === 'table') {
|
||||
var rows = el.rows || el.ROWS || [];
|
||||
if (rows.length > 0) {
|
||||
var ncols = (rows[0]||[]).length;
|
||||
var ncols = (rows[0] || []).length;
|
||||
textLines.push('--- Таблица ' + rows.length + '×' + ncols + ' ---');
|
||||
rows.forEach(function(row) {
|
||||
var cells = (row||[]).map(function(c){ return String(c||'').replace(/\n/g,' ').replace(/\|/g,'\\|'); });
|
||||
rows.forEach(function (row) {
|
||||
var cells = (row || []).map(function (c) { return String(c || '').replace(/\n/g, ' ').replace(/\|/g, '\\|'); });
|
||||
textLines.push('| ' + cells.join(' | ') + ' |');
|
||||
});
|
||||
textLines.push('');
|
||||
}
|
||||
}
|
||||
});
|
||||
var textify = textLines.join('\n').replace(/</g,'<').replace(/>/g,'>');
|
||||
var textify = textLines.join('\n').replace(/</g, '<').replace(/>/g, '>');
|
||||
|
||||
// Левая: сырой текст (textify)
|
||||
leftHtml = '<pre>' + textify + '</pre>';
|
||||
@@ -481,7 +486,7 @@ async function showText(i) {
|
||||
rightHtml += '<div class="kv"><span class="k">Изменён</span><span class="v">' + formatDate(f.lastModified) + '</span></div>';
|
||||
if (f.parseInfo) {
|
||||
var d = f.parseInfo;
|
||||
rightHtml += '<div class="kv"><span class="k">Элементов</span><span class="v">' + (d.element_count||0) + '</span></div>';
|
||||
rightHtml += '<div class="kv"><span class="k">Элементов</span><span class="v">' + (d.element_count || 0) + '</span></div>';
|
||||
if (d.error) rightHtml += '<div style="margin-top:8px;color:var(--destructive);">' + d.error + '</div>';
|
||||
}
|
||||
rightHtml += '<div style="margin-top:10px;font-weight:600;font-size:12px;">Все элементы:</div>';
|
||||
@@ -499,13 +504,13 @@ async function showText(i) {
|
||||
if (qData.doc_date) rightHtml += '<div class="kv"><span class="k">Дата</span><span class="v">' + escHtml(qData.doc_date) + '</span></div>';
|
||||
if (qData.counterparty) rightHtml += '<div class="kv"><span class="k">Контрагент</span><span class="v">' + escHtml(qData.counterparty) + '</span></div>';
|
||||
if (qData.classify_raw) {
|
||||
var rawId = 'raw_' + docId.replace(/-/g,'');
|
||||
var rawId = 'raw_' + docId.replace(/-/g, '');
|
||||
rightHtml += '<details style="margin-top:8px;"><summary style="cursor:pointer;font-size:11px;color:var(--brand-primary);">Сырой ответ LLM</summary>';
|
||||
rightHtml += '<pre style="font-size:10px;max-height:200px;overflow:auto;background:var(--brand-grey-light);padding:6px;border-radius:4px;margin-top:4px;">' + escHtml(qData.classify_raw) + '</pre>';
|
||||
rightHtml += '</details>';
|
||||
}
|
||||
}
|
||||
} catch(e) {
|
||||
} catch (e) {
|
||||
leftHtml = '<span style="color:var(--destructive);">Ошибка</span>';
|
||||
}
|
||||
} else {
|
||||
@@ -535,11 +540,11 @@ var promptNotes = document.getElementById('promptNotes');
|
||||
// extract — извлечение строк из первого документа (базовый договор)
|
||||
// diff — сравнение допсоглашений (ADD/UPDATE/DELETE)
|
||||
// classify — классификация документа (тип, номер, контрагент, дата)
|
||||
var activePromptId = {extract: '', diff: '', classify: ''};
|
||||
var activePromptId = { extract: '', diff: '', classify: '' };
|
||||
|
||||
function openPromptModal(role) {
|
||||
promptRole = role;
|
||||
var labels = {extract: 'Извлечение', diff: 'Сравнение', classify: 'Классификация'};
|
||||
var labels = { extract: 'Извлечение', diff: 'Сравнение', classify: 'Классификация' };
|
||||
promptModalTitle.textContent = 'Промпт: ' + (labels[role] || role);
|
||||
document.getElementById('promptTabExtract').style.background = role === 'extract' ? 'var(--brand-primary)' : '';
|
||||
document.getElementById('promptTabExtract').style.color = role === 'extract' ? '#fff' : '';
|
||||
@@ -561,26 +566,26 @@ function closePromptModal(e) {
|
||||
|
||||
function loadPrompt(role) {
|
||||
fetch(VM_API + '/api/prompts?role=' + role)
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(d) {
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (d) {
|
||||
if (d.ok) {
|
||||
promptEditor.value = d.body || '';
|
||||
activePromptId[role] = d.id || '';
|
||||
loadHistory(role);
|
||||
}
|
||||
})
|
||||
.catch(function(e) { console.error('loadPrompt:', e); });
|
||||
.catch(function (e) { console.error('loadPrompt:', e); });
|
||||
}
|
||||
|
||||
function loadHistory(role) {
|
||||
fetch(VM_API + '/api/prompts/list?role=' + role)
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(d) {
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (d) {
|
||||
if (!d.ok || !d.versions || !d.versions.length) {
|
||||
promptHistory.innerHTML = '<div style="color:var(--muted);">нет версий</div>';
|
||||
return;
|
||||
}
|
||||
promptHistory.innerHTML = d.versions.map(function(v) {
|
||||
promptHistory.innerHTML = d.versions.map(function (v) {
|
||||
var isActive = v.is_active ? '<span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:var(--green);margin-right:4px;" title="активная"></span> ' : '';
|
||||
var style = v.is_active ? 'font-weight:600;' : '';
|
||||
return '<div style="display:flex;align-items:center;padding:4px 0;border-bottom:1px solid var(--brand-gray);font-size:11px;' + style + '">'
|
||||
@@ -592,17 +597,17 @@ function loadHistory(role) {
|
||||
+ '</div>';
|
||||
}).join('');
|
||||
})
|
||||
.catch(function(e) { console.error('loadHistory:', e); });
|
||||
.catch(function (e) { console.error('loadHistory:', e); });
|
||||
}
|
||||
|
||||
function activatePrompt(id) {
|
||||
fetch(VM_API + '/api/prompts/activate', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({id: id})
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ id: id })
|
||||
})
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(d) {
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (d) {
|
||||
if (d.ok) { loadPrompt(promptRole); promptStatus.textContent = '✓ активирован'; }
|
||||
else { promptStatus.textContent = '✕ ' + (d.error || 'ошибка'); }
|
||||
});
|
||||
@@ -612,30 +617,30 @@ function deletePrompt(id) {
|
||||
if (!confirm('Удалить эту версию промпта?')) return;
|
||||
fetch(VM_API + '/api/prompts/delete', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({id: id})
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ id: id })
|
||||
})
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(d) {
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (d) {
|
||||
if (d.ok) { loadHistory(promptRole); promptStatus.textContent = '✓ удалено'; }
|
||||
else { promptStatus.textContent = '✕ ' + (d.error || 'ошибка'); }
|
||||
});
|
||||
}
|
||||
|
||||
document.getElementById('promptSave').addEventListener('click', function() {
|
||||
document.getElementById('promptSave').addEventListener('click', function () {
|
||||
var body = promptEditor.value.trim();
|
||||
if (!body) { promptStatus.textContent = '✕ тело промпта пустое'; return; }
|
||||
var name = promptName.value.trim() || ('v' + new Date().toISOString().replace(/[:.]/g,'-').substring(0,16));
|
||||
var name = promptName.value.trim() || ('v' + new Date().toISOString().replace(/[:.]/g, '-').substring(0, 16));
|
||||
fetch(VM_API + '/api/prompts/save', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
role: promptRole, name: name, body: body,
|
||||
notes: promptNotes.value.trim(), is_active: true
|
||||
})
|
||||
})
|
||||
.then(function(r) { return r.json(); })
|
||||
.then(function(d) {
|
||||
.then(function (r) { return r.json(); })
|
||||
.then(function (d) {
|
||||
if (d.ok) {
|
||||
promptStatus.textContent = '✓ сохранено (новая версия)';
|
||||
promptName.value = ''; promptNotes.value = '';
|
||||
@@ -646,10 +651,10 @@ document.getElementById('promptSave').addEventListener('click', function() {
|
||||
});
|
||||
});
|
||||
|
||||
document.getElementById('promptTabExtract').addEventListener('click', function() { openPromptModal('extract'); });
|
||||
document.getElementById('promptTabDiff').addEventListener('click', function() { openPromptModal('diff'); });
|
||||
document.getElementById('promptTabClassify').addEventListener('click', function() { openPromptModal('classify'); });
|
||||
document.getElementById('promptBtnExtract').addEventListener('click', function() { openPromptModal('extract'); });
|
||||
document.getElementById('promptBtnDiff').addEventListener('click', function() { openPromptModal('diff'); });
|
||||
document.getElementById('promptBtnClassify').addEventListener('click', function() { openPromptModal('classify'); });
|
||||
document.getElementById('promptTabExtract').addEventListener('click', function () { openPromptModal('extract'); });
|
||||
document.getElementById('promptTabDiff').addEventListener('click', function () { openPromptModal('diff'); });
|
||||
document.getElementById('promptTabClassify').addEventListener('click', function () { openPromptModal('classify'); });
|
||||
document.getElementById('promptBtnExtract').addEventListener('click', function () { openPromptModal('extract'); });
|
||||
document.getElementById('promptBtnDiff').addEventListener('click', function () { openPromptModal('diff'); });
|
||||
document.getElementById('promptBtnClassify').addEventListener('click', function () { openPromptModal('classify'); });
|
||||
lucide.createIcons();
|
||||
|
||||
Reference in New Issue
Block a user