diff --git a/app/app.py b/app/app.py index e7b565b..0a7e4a5 100644 --- a/app/app.py +++ b/app/app.py @@ -1,128 +1,64 @@ -"""contracts-flask — Flask-фронтенд для сверки договоров. -1:1 функционал с Lucee-версией. -Все тяжёлые операции — на ВМ (contracts.kube5s.ru). +"""contracts-flask v2.0 — полный перенос с ВМ на Flask. +Больше никаких прокси на contracts.kube5s.ru — всё локально. """ -import os -import httpx -from flask import Flask, render_template, request, jsonify - -app = Flask(__name__) - -VM_API = os.environ.get("VM_API", "https://contracts.kube5s.ru") -LLM_URL = os.environ.get("LLM_API_URL", "https://api.aillm.ru/v1/chat/completions") -LLM_KEY = os.environ.get("LLM_API_KEY", "") -LLM_MODEL = os.environ.get("LLM_MODEL", "gpt-oss-120b") +from flask import Flask +from app.config import VERSION, MAX_CONTENT_LENGTH +from app.routes import register_routes -@app.route("/") -def index(): - return render_template("index.html", vm_api=VM_API) +def create_app(): + app = Flask(__name__) + app.config["VERSION"] = VERSION + app.config["MAX_CONTENT_LENGTH"] = MAX_CONTENT_LENGTH + + # DB auto-seed — schema migration + seed prompts + crash recovery + _init_db() + + # Регистрация всех blueprint'ов + register_routes(app) + + # no_cache на все ответы + @app.after_request + def no_cache(response): + response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" + response.headers["Pragma"] = "no-cache" + response.headers["Expires"] = "0" + return response + + return app -@app.route("/chat", methods=["POST"]) -def chat(): - contract_id = request.args.get("contract_id", "") - question = request.form.get("question", "") - if not contract_id or not question: - return jsonify({"ok": False, "error": "contract_id and question required"}) - +def _init_db(): + """Schema migration + recovery: сброс застрявших classify.""" try: - with httpx.Client(timeout=10) as client: - resp = client.get(f"{VM_API}/api/spec-current", params={"contract_id": contract_id}) - resp.raise_for_status() - rows = resp.json().get("rows", []) - except Exception as e: - return jsonify({"ok": False, "error": f"VM error: {e}"}) + from app.db.connection import execute - if not rows: - return jsonify({"ok": True, "answer": "Нет данных. Сначала запустите сравнение."}) + for sql in [ + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_raw text", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_input text", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS zip_source text", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS content_hash text", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS batch_id uuid", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_status text DEFAULT 'pending'", + "ALTER TABLE documents ALTER COLUMN classify_status SET DEFAULT 'pending'", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS doc_type text", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS own_number text", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS parent_number text", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS doc_date date", + "ALTER TABLE documents ADD COLUMN IF NOT EXISTS counterparty text", + ]: + execute(sql) - ctx = "\n".join(f"{r.get('name','')} | цена={r.get('price')} | объём={r.get('qty')} | сумма={r.get('sum')} | начало={r.get('date_start')}" for r in rows) - prompt = f"Ты — анализатор договоров. Данные:\n{ctx}\n\nВопрос: {question}\nОтветь кратко, только по данным." + from app.db import prompts as db_prompts + db_prompts.seed_defaults() - try: - with httpx.Client(http2=True, timeout=120) as client: - resp = client.post(LLM_URL, json={ - "model": LLM_MODEL, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": 1000, "temperature": 0.1, - }, headers={"Authorization": f"Bearer {LLM_KEY}", "Content-Type": "application/json"}) - resp.raise_for_status() - return jsonify({"ok": True, "answer": resp.json()["choices"][0]["message"]["content"]}) - except Exception as e: - return jsonify({"ok": False, "error": f"LLM error: {e}"}) + # Crash recovery: сбросить застрявшие classify + execute( + "UPDATE documents SET classify_status='pending', error_message=NULL " + "WHERE classify_status='processing'" + ) + except Exception: + pass # БД недоступна при старте — не фатально -@app.route("/api/prompts", methods=["GET"]) -def prompts_get(): - try: - with httpx.Client(timeout=10) as client: - resp = client.get(f"{VM_API}/api/prompts", params=request.args) - return jsonify(resp.json()) - except Exception as e: - return jsonify({"ok": False, "error": str(e)}) - - -@app.route("/api/prompts/list", methods=["GET"]) -def prompts_list(): - try: - with httpx.Client(timeout=10) as client: - resp = client.get(f"{VM_API}/api/prompts/list") - return jsonify(resp.json()) - except Exception as e: - return jsonify({"ok": False, "error": str(e)}) - - -@app.route("/api/prompts/save", methods=["POST"]) -def prompts_save(): - try: - with httpx.Client(timeout=10) as client: - resp = client.post(f"{VM_API}/api/prompts/save", json=request.get_json()) - return jsonify(resp.json()) - except Exception as e: - return jsonify({"ok": False, "error": str(e)}) - - -@app.route("/api/prompts/activate", methods=["POST"]) -def prompts_activate(): - try: - with httpx.Client(timeout=10) as client: - resp = client.post(f"{VM_API}/api/prompts/activate", json=request.get_json()) - return jsonify(resp.json()) - except Exception as e: - return jsonify({"ok": False, "error": str(e)}) - - -@app.route("/architect") -def architect(): - return render_template("architect.html") - - -@app.route("/ci-cd") -def ci_cd(): - return render_template("ci-cd.html") - - -@app.route("/drhider") -@app.route("/DrHider") -def drhider(): - return render_template("drhider.html", vm_api=VM_API) - - -@app.route("/health") -def health(): - return {"ok": True, "service": "contracts-flask"} - - -if __name__ == "__main__": - app.run(host="0.0.0.0", port=5000, threaded=True) - -# ── Совместимость с Lucee-путями ────────────────────────────── - -@app.route("/chat.cfm", methods=["POST"]) -def chat_cfm(): - return chat() - -@app.route("/prompt.cfm", methods=["GET"]) -def prompt_cfm(): - return prompts_get() +app = create_app() diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..3217fb6 --- /dev/null +++ b/app/config.py @@ -0,0 +1,19 @@ +"""Конфигурация приложения — все настройки в одном месте.""" +import os + +VERSION = "2.0.0" + +DB_CONFIG = { + "host": os.getenv("DB_HOST", "127.0.0.1"), + "port": int(os.getenv("DB_PORT", "5432")), + "dbname": os.getenv("DB_NAME", "baza"), + "user": os.getenv("DB_USER", "super"), + "password": os.getenv("DB_PASS", ""), +} + +LLM_URL = os.getenv("LLM_API_URL", "https://api.aillm.ru/v1/chat/completions") +LLM_KEY = os.getenv("LLM_API_KEY", "") +LLM_MODEL = os.getenv("LLM_MODEL", "gpt-oss-120b") + +MAX_CONTENT_LENGTH = 200 * 1024 * 1024 # 200 MB +API_KEY = os.getenv("API_KEY", "") diff --git a/app/routes/__init__.py b/app/routes/__init__.py new file mode 100644 index 0000000..0cc591e --- /dev/null +++ b/app/routes/__init__.py @@ -0,0 +1,17 @@ +"""Регистрация всех blueprint'ов.""" + + +def register_routes(app): + from app.routes.upload_bp import upload_bp + from app.routes.pipeline_bp import pipeline_bp + from app.routes.api_bp import api_bp + from app.routes.prompts_bp import prompts_bp + from app.routes.health_bp import health_bp + from app.routes.pages_bp import pages_bp + + app.register_blueprint(upload_bp) + app.register_blueprint(pipeline_bp) + app.register_blueprint(api_bp) + app.register_blueprint(prompts_bp) + app.register_blueprint(health_bp) + app.register_blueprint(pages_bp) diff --git a/app/routes/api_bp.py b/app/routes/api_bp.py new file mode 100644 index 0000000..81269c3 --- /dev/null +++ b/app/routes/api_bp.py @@ -0,0 +1,150 @@ +"""API blueprint — groups, documents, supplements, sync, cleanup, spec-current.""" +from flask import Blueprint, request, jsonify +from app.db import documents, supplements, spec_current +from app.db.connection import execute, query +from app.services.grouping import group_documents, apply_groups +from app.config import API_KEY + +api_bp = Blueprint("api", __name__) + + +def _require_auth(): + """Проверка API-ключа для опасных операций.""" + if API_KEY: + key = request.headers.get("X-Api-Key", "") + if key != API_KEY: + return False + return True + + +# ── Supplements ────────────────────────────────────────────────── + +@api_bp.route("/api/supplements") +def api_supplements(): + cid = request.args.get("contract_id") + if not cid: + return jsonify(ok=False, error="contract_id required"), 400 + rows = supplements.list_by_contract(cid) + return jsonify(ok=True, supplements=rows) + + +# ── Documents ──────────────────────────────────────────────────── + +@api_bp.route("/api/documents/") +def api_document(doc_id): + doc = documents.get(doc_id) + if not doc: + return jsonify(ok=False, error="not found"), 404 + return jsonify(ok=True, **{ + k: doc.get(k) for k in [ + "id", "filename", "status", "elements_json", "doc_type", + "own_number", "parent_number", "doc_date", "counterparty", + "classify_status", "classify_raw", "classify_input", + ] + }) + + +@api_bp.route("/api/documents/", methods=["DELETE"]) +def api_document_delete(doc_id): + supps = query("SELECT id, contract_id FROM supplements WHERE document_id = %s", (doc_id,)) + for s in (supps or []): + execute( + """DELETE FROM spec_current WHERE contract_id = %s + AND last_event_id IN (SELECT id FROM spec_events WHERE supplement_id = %s)""", + (s["contract_id"], s["id"]), + ) + execute("DELETE FROM spec_events WHERE supplement_id = %s", (s["id"],)) + execute("DELETE FROM supplements WHERE id = %s", (s["id"],)) + execute("DELETE FROM documents WHERE id = %s", (doc_id,)) + return jsonify(ok=True) + + +# ── Sync ───────────────────────────────────────────────────────── + +@api_bp.route("/api/sync", methods=["POST"]) +def api_sync(): + """Удалить документы, не входящие в keep_ids.""" + body = request.get_json() + keep_ids = set(body.get("keep_ids", [])) + if len(keep_ids) > 1000: + return jsonify(ok=False, error="too many keep_ids (max 1000)"), 400 + + docs = query("SELECT id FROM documents", ()) + deleted = 0 + for d in (docs or []): + if d["id"] in keep_ids: + continue + supps = query("SELECT id, contract_id FROM supplements WHERE document_id = %s", (d["id"],)) + for s in (supps or []): + execute( + """DELETE FROM spec_current WHERE contract_id = %s + AND last_event_id IN (SELECT id FROM spec_events WHERE supplement_id = %s)""", + (s["contract_id"], s["id"]), + ) + execute("DELETE FROM spec_events WHERE supplement_id = %s", (s["id"],)) + execute("DELETE FROM supplements WHERE id = %s", (s["id"],)) + execute("DELETE FROM documents WHERE id = %s", (d["id"],)) + deleted += 1 + + execute("DELETE FROM contracts WHERE id NOT IN (SELECT DISTINCT contract_id FROM supplements)") + return jsonify(ok=True, deleted=deleted) + + +# ── Groups ─────────────────────────────────────────────────────── + +@api_bp.route("/api/groups") +def api_groups(): + batch_id = request.args.get("batch") + if not batch_id: + return jsonify(ok=False, error="batch required"), 400 + result = group_documents(batch_id) + return jsonify(result) + + +@api_bp.route("/api/batch-progress") +def api_batch_progress(): + batch_id = request.args.get("batch") + if not batch_id: + return jsonify(ok=False, error="batch required"), 400 + counts = documents.count_by_status(batch_id) + docs = documents.list_by_batch(batch_id) + return jsonify(ok=True, counts=counts, total=len(docs)) + + +@api_bp.route("/api/apply-groups", methods=["POST"]) +def api_apply_groups(): + body = request.get_json() + batch_id = body.get("batch_id") + groups = body.get("groups", []) + if not batch_id: + return jsonify(ok=False, error="batch_id required"), 400 + result = apply_groups(batch_id, groups) + return jsonify(result) + + +# ── Spec current ───────────────────────────────────────────────── + +@api_bp.route("/api/spec-current") +def api_spec_current(): + cid = request.args.get("contract_id") + if not cid: + return jsonify(ok=False, error="contract_id required"), 400 + rows = spec_current.list_by_contract(cid) + return jsonify(ok=True, rows=rows) + + +# ── Cleanup (требует API-ключ) ──────────────────────────────────── + +@api_bp.route("/api/cleanup", methods=["POST"]) +def api_cleanup(): + """Полная очистка всех данных (кроме prompts).""" + if not _require_auth(): + return jsonify(ok=False, error="unauthorized"), 401 + + execute("DELETE FROM spec_current") + execute("DELETE FROM spec_events") + execute("DELETE FROM supplements") + execute("DELETE FROM upload_chunks") + execute("DELETE FROM documents") + execute("DELETE FROM contracts") + return jsonify(ok=True, message="all data cleaned") diff --git a/app/routes/health_bp.py b/app/routes/health_bp.py new file mode 100644 index 0000000..fa632bd --- /dev/null +++ b/app/routes/health_bp.py @@ -0,0 +1,10 @@ +"""Health probe — обязательно для Штурвала.""" +from flask import Blueprint, jsonify +from app.config import VERSION + +health_bp = Blueprint("health", __name__) + + +@health_bp.route("/health") +def health(): + return jsonify({"ok": True, "version": VERSION}) diff --git a/app/routes/pages_bp.py b/app/routes/pages_bp.py new file mode 100644 index 0000000..09b17a5 --- /dev/null +++ b/app/routes/pages_bp.py @@ -0,0 +1,14 @@ +"""HTML-страницы.""" +from flask import Blueprint, render_template + +pages_bp = Blueprint("pages", __name__) + + +@pages_bp.route("/") +def index(): + return render_template("index.html") + + +@pages_bp.route("/architect") +def architect(): + return render_template("architect.html") diff --git a/app/routes/pipeline_bp.py b/app/routes/pipeline_bp.py new file mode 100644 index 0000000..d282861 --- /dev/null +++ b/app/routes/pipeline_bp.py @@ -0,0 +1,91 @@ +"""Pipeline blueprint — SSE-сравнение + classify.""" +import json, re, os, threading +from flask import Blueprint, request, jsonify, Response, stream_with_context +from app.services.process import run_pipeline +from app.services.classify import classify_batch +from app.llm_prompt import build_prompt +from app.db import documents + +pipeline_bp = Blueprint("pipeline", __name__) + +# In-memory lock для classify (замена файлового lock) +_classify_locks: dict[str, threading.Thread] = {} + + +@pipeline_bp.route("/process-v2", methods=["GET"]) +def process_v2(): + """SSE-стриминг сравнения договоров.""" + cid = request.args.get("contract_id") + if not cid or not re.fullmatch( + r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', cid, re.I + ): + return jsonify(ok=False, error="invalid contract_id"), 400 + + order_ids = request.args.get("order", "") + + def generate(): + # Heartbeat каждые 15 сек — держит соединение при медленном LLM + import time as _time + last_beat = _time.time() + + yield ": ok\n\n" + try: + for event in run_pipeline(cid, order_ids, build_prompt): + now = _time.time() + if now - last_beat >= 15: + yield ": heartbeat\n\n" + last_beat = now + yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n" + except GeneratorExit: + return + except Exception as e: + yield f"data: {json.dumps({'type': 'error', 'message': str(e)}, ensure_ascii=False)}\n\n" + + return Response( + stream_with_context(generate()), + content_type="text/event-stream; charset=utf-8", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + }, + ) + + +@pipeline_bp.route("/api/classify-batch", methods=["POST"]) +def classify_batch_route(): + """Запустить классификацию. ≤10 файлов — sync, >10 — async (Thread).""" + body = request.get_json() + batch_id = body.get("batch_id") + if not batch_id: + return jsonify(ok=False, error="batch_id required"), 400 + + # Guard: уже запущена? + if batch_id in _classify_locks: + t = _classify_locks[batch_id] + if t.is_alive(): + return jsonify(ok=False, error="classify already running"), 409 + else: + del _classify_locks[batch_id] + + pending = documents.list_pending(batch_id) + total = len(pending) + if total == 0: + return jsonify(ok=False, error="no pending documents"), 400 + + # Sync для малых батчей + if total <= 10: + result = classify_batch(batch_id) + return jsonify(result) + + # Async для больших + def _run(): + try: + classify_batch(batch_id) + finally: + _classify_locks.pop(batch_id, None) + + t = threading.Thread(target=_run, daemon=True) + _classify_locks[batch_id] = t + t.start() + + return jsonify(ok=True, total=total), 202 diff --git a/app/routes/prompts_bp.py b/app/routes/prompts_bp.py new file mode 100644 index 0000000..12bb94d --- /dev/null +++ b/app/routes/prompts_bp.py @@ -0,0 +1,71 @@ +"""Prompts blueprint — CRUD + activate для версионирования промптов.""" +from flask import Blueprint, request, jsonify +from app.db import prompts as db_prompts + +prompts_bp = Blueprint("prompts", __name__) + + +@prompts_bp.route("/api/prompts", methods=["GET"]) +def prompts_get(): + role = request.args.get("role") + if not role: + return jsonify(ok=False, error="role required"), 400 + p = db_prompts.get_active(role) + if p: + return jsonify(ok=True, **{ + "id": p["id"], "role": p["role"], "name": p["name"], + "body": p["body"], "is_active": p["is_active"], + "notes": p.get("notes"), "created_at": str(p.get("created_at", "")), + }) + return jsonify(ok=False, error="not found"), 404 + + +@prompts_bp.route("/api/prompts/list", methods=["GET"]) +def prompts_list(): + role = request.args.get("role") + if not role: + return jsonify(ok=False, error="role required"), 400 + versions = db_prompts.list_by_role(role) + return jsonify(ok=True, versions=versions) + + +@prompts_bp.route("/api/prompts/save", methods=["POST"]) +def prompts_save(): + body = request.get_json() + if not body: + return jsonify(ok=False, error="body required"), 400 + role = body.get("role", "") + name = body.get("name", "") + prompt_body = body.get("body", "") + notes = body.get("notes", "") + try: + p = db_prompts.insert(role, name, prompt_body, notes) + return jsonify(ok=True, id=p["id"]) + except Exception as e: + return jsonify(ok=False, error=str(e)), 500 + + +@prompts_bp.route("/api/prompts/activate", methods=["POST"]) +def prompts_activate(): + body = request.get_json() + if not body: + return jsonify(ok=False, error="body required"), 400 + prompt_id = body.get("id") + try: + db_prompts.activate(prompt_id) + return jsonify(ok=True) + except Exception as e: + return jsonify(ok=False, error=str(e)), 500 + + +@prompts_bp.route("/api/prompts/delete", methods=["POST"]) +def prompts_delete(): + body = request.get_json() + if not body: + return jsonify(ok=False, error="body required"), 400 + prompt_id = body.get("id") + try: + db_prompts.delete(prompt_id) + return jsonify(ok=True) + except Exception as e: + return jsonify(ok=False, error=str(e)), 500 diff --git a/app/routes/upload_bp.py b/app/routes/upload_bp.py new file mode 100644 index 0000000..c0b9237 --- /dev/null +++ b/app/routes/upload_bp.py @@ -0,0 +1,145 @@ +"""Upload blueprint — загрузка, конвертация, распаковка.""" +import io, os, base64, hashlib, zipfile, tempfile, subprocess +from flask import Blueprint, request, jsonify, send_file +from app.services.parse import parse_file +from app.db import documents +from app.config import MAX_CONTENT_LENGTH + +upload_bp = Blueprint("upload", __name__) + +ALLOWED = {"pdf", "docx", "doc", "zip"} + + +def _check_ext(filename: str) -> str | None: + ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else "" + if ext not in ALLOWED: + return f"unsupported format: .{ext} (allowed: {', '.join(sorted(ALLOWED))})" + return None + + +@upload_bp.route("/upload", methods=["POST"]) +def upload(): + """Загрузка одного файла + авто-парсинг → БД.""" + f = request.files.get("files") + if not f: + return jsonify(ok=False, error="no file"), 400 + + err = _check_ext(f.filename) + if err: + return jsonify(ok=False, error=err), 400 + + data = f.read() + content_hash = hashlib.sha256(data).hexdigest()[:16] + batch_id = request.form.get("batch_id") + zip_source = request.form.get("zip_source") + + # Дедупликация по хешу + if batch_id: + existing = documents.get_by_hash(batch_id, content_hash) + if existing: + return jsonify(ok=False, error="duplicate", doc_id=existing["id"]) + + doc = documents.insert( + filename=f.filename, + mime_type=f.content_type or "application/octet-stream", + original_bytes=base64.b64encode(data).decode(), + batch_id=batch_id, + zip_source=zip_source, + content_hash=content_hash, + ) + + # Авто-парсинг + try: + result = parse_file(f.filename, data) + if result["status"] == "parsed": + documents.set_parsed(doc["id"], result["elements"]) + else: + documents.set_error(doc["id"], result.get("error", "parse failed")) + except Exception as e: + documents.set_error(doc["id"], str(e)) + result = {"status": "error", "error": str(e)} + + contract_id = request.form.get("contract_id") + return jsonify( + ok=True, + doc_id=doc["id"], + contract_id=contract_id, + parsed={"status": result["status"], "element_count": result.get("element_count", 0)}, + ) + + +@upload_bp.route("/convert-doc", methods=["POST"]) +def convert_doc(): + """.doc → .docx через libreoffice (без сохранения на диск).""" + f = request.files.get("files") + if not f: + return jsonify(ok=False, error="no file"), 400 + + data = f.read() + doc_path = None + tmpdir = None + try: + with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as tmp: + tmp.write(data) + doc_path = tmp.name + tmpdir = tempfile.mkdtemp() + subprocess.run( + ["libreoffice", "--headless", "--convert-to", "docx", "--outdir", tmpdir, doc_path], + timeout=30, capture_output=True, + ) + docx_files = [x for x in os.listdir(tmpdir) if x.endswith(".docx")] + if docx_files: + with open(os.path.join(tmpdir, docx_files[0]), "rb") as out: + return send_file( + io.BytesIO(out.read()), + mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ) + return jsonify(ok=False, error="conversion produced no output"), 500 + finally: + if doc_path and os.path.exists(doc_path): + os.unlink(doc_path) + if tmpdir and os.path.exists(tmpdir): + for x in os.listdir(tmpdir): + os.unlink(os.path.join(tmpdir, x)) + os.rmdir(tmpdir) + + +@upload_bp.route("/unzip-upload", methods=["POST"]) +def unzip_upload(): + """Распаковать ZIP → список файлов (base64 для фронтенда).""" + f = request.files.get("files") + if not f: + return jsonify(ok=False, error="no file"), 400 + + data = f.read() + MAX_FILES = 500 + MAX_UNCOMPRESSED = 500 * 1024 * 1024 # 500 MB + + files = [] + total = 0 + + with zipfile.ZipFile(io.BytesIO(data)) as zf: + if len(zf.namelist()) > MAX_FILES: + return jsonify(ok=False, error=f"too many files in ZIP (max {MAX_FILES})"), 400 + + for info in zf.infolist(): + if info.is_dir(): + continue + name = os.path.basename(info.filename) + if not name or ".." in name or "/" in name or "\\" in name: + continue + + raw = zf.read(info) + total += len(raw) + if total > MAX_UNCOMPRESSED: + return jsonify(ok=False, error="total uncompressed size exceeds 500 MB"), 400 + + ext = name.rsplit(".", 1)[-1].lower() if "." in name else "" + files.append({ + "filename": name, + "ext": ext, + "size": len(raw), + "data_b64": base64.b64encode(raw).decode(), + }) + + return jsonify(ok=True, files=files) diff --git a/app/services/process.py b/app/services/process.py new file mode 100644 index 0000000..9c3ace5 --- /dev/null +++ b/app/services/process.py @@ -0,0 +1,155 @@ +"""Process service — SSE pipeline: reset → supplements → LLM → apply. +Адаптирован из deploy/compare/process.py: callback → generator. +""" +import json, time +from app.db import supplements, spec_current, spec_events +from app.services.metrics import check_arithmetic + + +def run_pipeline(contract_id, order_ids, build_prompt_fn): + """Generator: yield SSE events вместо sse_send callback. + + Использование: + for event in run_pipeline(cid, order_ids, build_prompt): + yield f"data: {json.dumps(event)}\\n\\n" + """ + t0 = time.time() + + # 0. Reset + spec_events.reset(contract_id) + + # 1. Get supplements with parsed documents + supps = supplements.list_by_contract(contract_id) + if order_ids: + order_list = [x.strip() for x in order_ids.split(",") if x.strip()] + order_map = {oid: i for i, oid in enumerate(order_list)} + supps.sort(key=lambda s: order_map.get(s["id"], 999999)) + else: + supps.sort(key=lambda s: (s.get("doc_date") or "9999-99-99", s.get("created_at", ""))) + + if not supps: + yield {"type": "error", "message": "Нет распарсенных файлов"} + return + + from app.services.llm import call_llm + + for s in supps: + sid = s["id"] + filename = s.get("filename", "?") + + # Current spec + cur = spec_current.list_by_contract(contract_id) + current_spec = [] + for r in cur: + current_spec.append({ + "hash": r["name_hash"], + "name": r["name"], + "price": float(r["price"]) if r.get("price") is not None else None, + "qty": float(r["qty"]) if r.get("qty") is not None else None, + "sum": float(r["sum"]) if r.get("sum") is not None else None, + "date_start": r["date_start"], + }) + + # Get elements_json + ej = spec_current.get_elements_json(s["document_id"]) + if not ej: + yield { + "type": "extract_error", + "supplement_id": sid, + "filename": filename, + "error": "no elements_json", + } + continue + + # Build doc text from elements + doc_text = _elements_to_text(ej) + + yield { + "type": "extract_start", + "supplement_id": sid, + "filename": filename, + } + + # LLM call + try: + t1 = time.time() + result, prompt_id = call_llm(current_spec, doc_text, build_prompt_fn) + ops = result.get("ops", []) + mode = result.get("mode", "llm") + + yield { + "type": "llm_done", + "supplement_id": sid, + "filename": filename, + "ops_count": len(ops), + "mode": mode, + "time_s": round(time.time() - t1, 1), + } + + # Apply ops to DB + applied_ops = [] + summary = {"added": 0, "updated": 0, "deleted": 0, "unresolved": 0} + for op in ops: + action = op.get("action", "UNRESOLVED") + summary[action.lower()] = summary.get(action.lower(), 0) + 1 + try: + if action == "ADD": + nr = op.get("new_row", {}) + spec_events.add_row(contract_id, sid, nr) + elif action == "UPDATE": + nr = op.get("new_row", {}) + nv = op.get("new_values", {}) + target = op.get("target_hash", "") + spec_events.update_row(contract_id, sid, target, nr, nv) + elif action == "DELETE": + target = op.get("target_hash", "") + spec_events.delete_row(contract_id, sid, target) + applied_ops.append(op) + except Exception: + summary["unresolved"] = summary.get("unresolved", 0) + 1 + + # Arithmetic check + check_arithmetic(contract_id) + + yield { + "type": "applied", + "supplement_id": sid, + "filename": filename, + "summary": summary, + "ops": applied_ops, + } + + except Exception as e: + yield { + "type": "extract_error", + "supplement_id": sid, + "filename": filename, + "error": str(e), + } + + total_time = round(time.time() - t0, 1) + yield {"type": "complete", "total_time_s": total_time} + + +def _elements_to_text(ej): + """Extract flat text from elements_json for LLM prompt.""" + if isinstance(ej, dict) and "Value" in ej: + ej = ej["Value"] + if isinstance(ej, str): + try: + ej = json.loads(ej) + except (json.JSONDecodeError, TypeError): + return ej + + lines = [] + if isinstance(ej, list): + for el in ej: + if isinstance(el, dict): + if el.get("type") == "paragraph": + lines.append(el.get("text", "")) + elif el.get("type") == "table": + for row in el.get("rows", []): + lines.append(" | ".join(str(c) for c in row)) + elif isinstance(el, str): + lines.append(el) + return "\n".join(lines) diff --git a/app/static/app.js b/app/static/app.js index 36a8b0a..0cd9e4f 100644 --- a/app/static/app.js +++ b/app/static/app.js @@ -1,10 +1,9 @@ // ⛔ НЕ МЕНЯТЬ БЕЗ РАЗРЕШЕНИЯ НАЕЛЯ ⛔ -// Contracts App — весь JS на VM (contracts.kube5s.ru) -// Lucee: только домен + index.cfm-скелет -var VM_API = 'https://check.kube5s.ru'; -var UPLOAD_URL = VM_API + '/upload'; -var CONVERT_URL = VM_API + '/convert-doc'; -var UNZIP_URL = VM_API + '/unzip-upload'; +// Contracts App v2.0 — всё на Flask, ВМ больше нет +var VM_API = ''; +var UPLOAD_URL = '/upload'; +var CONVERT_URL = '/convert-doc'; +var UNZIP_URL = '/unzip-upload'; // SITE_URL удалён (Фаза 4) — не использовался var fileInput = document.getElementById('fileInput'); @@ -328,7 +327,7 @@ document.getElementById('llmBtn').addEventListener('click', function() { compareStatus.textContent = '⏳ 0.0с'; var order = state.files.map(function(f) { return f.supp_id; }).filter(Boolean).join(','); - var url = 'https://check.kube5s.ru/process-v2?contract_id=' + state.contractId + + var url = '/process-v2?contract_id=' + state.contractId + (order ? '&order=' + encodeURIComponent(order) : ''); // Фаза 3: унифицированный SSE через startCompareSSE (compare.js) diff --git a/app/templates/index.html b/app/templates/index.html index d73ca6f..90f811a 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -73,7 +73,7 @@
Nubes - Сверка договоров — LLM AI-driven Event Sourcing v1.0.196-flask + Сверка договоров — LLM AI-driven Event Sourcing v2.0.0
○ Загрузка→ ○ Классификация→ diff --git a/requirements.txt b/requirements.txt index 2fa29fb..7513a80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ gunicorn httpx python-docx pdfplumber +psycopg2-binary