fix: site/ with __init__.py — proper Python package, no stdlib conflict

site/routes/__init__.py

@@ -0,0 +1,17 @@
+"""Регистрация всех blueprint'ов."""
+
+
+def register_routes(app):
+    from site.routes.upload_bp import upload_bp
+    from site.routes.pipeline_bp import pipeline_bp
+    from site.routes.api_bp import api_bp
+    from site.routes.prompts_bp import prompts_bp
+    from site.routes.health_bp import health_bp
+    from site.routes.pages_bp import pages_bp
+
+    app.register_blueprint(upload_bp)
+    app.register_blueprint(pipeline_bp)
+    app.register_blueprint(api_bp)
+    app.register_blueprint(prompts_bp)
+    app.register_blueprint(health_bp)
+    app.register_blueprint(pages_bp)

site/routes/api_bp.py

@@ -0,0 +1,182 @@
+"""API blueprint — groups, documents, supplements, sync, cleanup, spec-current, chat."""
+from flask import Blueprint, request, jsonify
+from site.db import documents, supplements, spec_current
+from site.db.connection import execute, query
+from site.services.grouping import group_documents, apply_groups
+from site.config import LLM_URL, LLM_KEY, LLM_MODEL
+import httpx
+
+api_bp = Blueprint("api", __name__)
+
+
+# ── Supplements ──────────────────────────────────────────────────
+
+@api_bp.route("/api/supplements")
+def api_supplements():
+    cid = request.args.get("contract_id")
+    if not cid:
+        return jsonify(ok=False, error="contract_id required"), 400
+    rows = supplements.list_by_contract(cid)
+    return jsonify(ok=True, supplements=rows)
+
+
+# ── Documents ────────────────────────────────────────────────────
+
+@api_bp.route("/api/documents/<doc_id>")
+def api_document(doc_id):
+    doc = documents.get(doc_id)
+    if not doc:
+        return jsonify(ok=False, error="not found"), 404
+    return jsonify(ok=True, **{
+        k: doc.get(k) for k in [
+            "id", "filename", "status", "elements_json", "doc_type",
+            "own_number", "parent_number", "doc_date", "counterparty",
+            "classify_status", "classify_raw", "classify_input",
+        ]
+    })
+
+
+@api_bp.route("/api/documents/<doc_id>", methods=["DELETE"])
+def api_document_delete(doc_id):
+    supps = query("SELECT id, contract_id FROM supplements WHERE document_id = %s", (doc_id,))
+    for s in (supps or []):
+        execute(
+            """DELETE FROM spec_current WHERE contract_id = %s
+               AND last_event_id IN (SELECT id FROM spec_events WHERE supplement_id = %s)""",
+            (s["contract_id"], s["id"]),
+        )
+        execute("DELETE FROM spec_events WHERE supplement_id = %s", (s["id"],))
+        execute("DELETE FROM supplements WHERE id = %s", (s["id"],))
+    execute("DELETE FROM documents WHERE id = %s", (doc_id,))
+    return jsonify(ok=True)
+
+
+# ── Sync ─────────────────────────────────────────────────────────
+
+@api_bp.route("/api/sync", methods=["POST"])
+def api_sync():
+    """Удалить документы, не входящие в keep_ids."""
+    body = request.get_json()
+    keep_ids = set(body.get("keep_ids", []))
+    if len(keep_ids) > 1000:
+        return jsonify(ok=False, error="too many keep_ids (max 1000)"), 400
+
+    docs = query("SELECT id FROM documents", ())
+    deleted = 0
+    for d in (docs or []):
+        if d["id"] in keep_ids:
+            continue
+        supps = query("SELECT id, contract_id FROM supplements WHERE document_id = %s", (d["id"],))
+        for s in (supps or []):
+            execute(
+                """DELETE FROM spec_current WHERE contract_id = %s
+                   AND last_event_id IN (SELECT id FROM spec_events WHERE supplement_id = %s)""",
+                (s["contract_id"], s["id"]),
+            )
+            execute("DELETE FROM spec_events WHERE supplement_id = %s", (s["id"],))
+            execute("DELETE FROM supplements WHERE id = %s", (s["id"],))
+        execute("DELETE FROM documents WHERE id = %s", (d["id"],))
+        deleted += 1
+
+    execute("DELETE FROM contracts WHERE id NOT IN (SELECT DISTINCT contract_id FROM supplements)")
+    return jsonify(ok=True, deleted=deleted)
+
+
+# ── Groups ───────────────────────────────────────────────────────
+
+@api_bp.route("/api/groups")
+def api_groups():
+    batch_id = request.args.get("batch")
+    if not batch_id:
+        return jsonify(ok=False, error="batch required"), 400
+    result = group_documents(batch_id)
+    return jsonify(result)
+
+
+@api_bp.route("/api/batch-progress")
+def api_batch_progress():
+    batch_id = request.args.get("batch")
+    if not batch_id:
+        return jsonify(ok=False, error="batch required"), 400
+    counts = documents.count_by_status(batch_id)
+    docs = documents.list_by_batch(batch_id)
+    return jsonify(ok=True, counts=counts, total=len(docs))
+
+
+@api_bp.route("/api/apply-groups", methods=["POST"])
+def api_apply_groups():
+    body = request.get_json()
+    batch_id = body.get("batch_id")
+    groups = body.get("groups", [])
+    if not batch_id:
+        return jsonify(ok=False, error="batch_id required"), 400
+    result = apply_groups(batch_id, groups)
+    return jsonify(result)
+
+
+# ── Spec current ─────────────────────────────────────────────────
+
+@api_bp.route("/api/spec-current")
+def api_spec_current():
+    cid = request.args.get("contract_id")
+    if not cid:
+        return jsonify(ok=False, error="contract_id required"), 400
+    rows = spec_current.list_by_contract(cid)
+    return jsonify(ok=True, rows=rows)
+
+
+# ── Chat (совместимость с Lucee /chat.cfm) ──────────────────────
+
+@api_bp.route("/chat", methods=["POST"])
+@api_bp.route("/chat.cfm", methods=["POST"])
+def chat():
+    """Чат с LLM по данным спецификации. Совместим с Lucee-форматом ответа."""
+    contract_id = request.args.get("contract_id", "")
+    question = request.form.get("question", "")
+    if not contract_id or not question:
+        return jsonify(OK=False, ERROR="contract_id and question required")
+
+    rows = spec_current.list_by_contract(contract_id)
+    if not rows:
+        return jsonify(OK=True, ANSWER="Нет данных. Сначала запустите сравнение.")
+
+    ctx = "\n".join(
+        f"{r.get('name','')} | цена={r.get('price')} | объём={r.get('qty')} | "
+        f"сумма={r.get('sum')} | начало={r.get('date_start')}"
+        for r in rows
+    )
+    prompt = (
+        f"Ты — анализатор договоров. Данные:\n{ctx}\n\n"
+        f"Вопрос: {question}\nОтветь кратко, только по данным."
+    )
+
+    try:
+        with httpx.Client(http2=True, timeout=120) as client:
+            resp = client.post(
+                LLM_URL,
+                json={
+                    "model": LLM_MODEL,
+                    "messages": [{"role": "user", "content": prompt}],
+                    "max_tokens": 1000,
+                    "temperature": 0.1,
+                },
+                headers={
+                    "Authorization": f"Bearer {LLM_KEY}",
+                    "Content-Type": "application/json",
+                },
+            )
+            resp.raise_for_status()
+            answer = resp.json()["choices"][0]["message"]["content"]
+            return jsonify(OK=True, ANSWER=answer)
+    except Exception as e:
+        return jsonify(OK=False, ERROR=f"LLM error: {e}")
+
+
+# ── Cleanup (атомарное удаление БД) ──────────────────────────────
+
+@api_bp.route("/api/cleanup", methods=["POST"])
+def api_cleanup():
+    """Полная очистка: os.remove(DB) + init новой. Данные гарантированно стёрты."""
+    from site.db.connection import cleanup_db
+    cleanup_db()
+    return jsonify(ok=True, message="all data cleaned")

site/routes/health_bp.py

@@ -0,0 +1,10 @@
+"""Health probe — обязательно для Штурвала."""
+from flask import Blueprint, jsonify
+from site.config import VERSION
+
+health_bp = Blueprint("health", __name__)
+
+
+@health_bp.route("/health")
+def health():
+    return jsonify({"ok": True, "version": VERSION})

site/routes/pages_bp.py

@@ -0,0 +1,14 @@
+"""HTML-страницы."""
+from flask import Blueprint, render_template
+
+pages_bp = Blueprint("pages", __name__)
+
+
+@pages_bp.route("/")
+def index():
+    return render_template("index.html")
+
+
+@pages_bp.route("/architect")
+def architect():
+    return render_template("architect.html")

site/routes/pipeline_bp.py

@@ -0,0 +1,91 @@
+"""Pipeline blueprint — SSE-сравнение + classify."""
+import json, re, os, threading
+from flask import Blueprint, request, jsonify, Response, stream_with_context
+from site.services.process import run_pipeline
+from site.services.classify import classify_batch
+from site.llm_prompt import build_prompt
+from site.db import documents
+
+pipeline_bp = Blueprint("pipeline", __name__)
+
+# In-memory lock для classify (замена файлового lock)
+_classify_locks: dict[str, threading.Thread] = {}
+
+
+@pipeline_bp.route("/process-v2", methods=["GET"])
+def process_v2():
+    """SSE-стриминг сравнения договоров."""
+    cid = request.args.get("contract_id")
+    if not cid or not re.fullmatch(
+        r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', cid, re.I
+    ):
+        return jsonify(ok=False, error="invalid contract_id"), 400
+
+    order_ids = request.args.get("order", "")
+
+    def generate():
+        # Heartbeat каждые 15 сек — держит соединение при медленном LLM
+        import time as _time
+        last_beat = _time.time()
+
+        yield ": ok\n\n"
+        try:
+            for event in run_pipeline(cid, order_ids, build_prompt):
+                now = _time.time()
+                if now - last_beat >= 15:
+                    yield ": heartbeat\n\n"
+                    last_beat = now
+                yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
+        except GeneratorExit:
+            return
+        except Exception as e:
+            yield f"data: {json.dumps({'type': 'error', 'message': str(e)}, ensure_ascii=False)}\n\n"
+
+    return Response(
+        stream_with_context(generate()),
+        content_type="text/event-stream; charset=utf-8",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Accel-Buffering": "no",
+        },
+    )
+
+
+@pipeline_bp.route("/api/classify-batch", methods=["POST"])
+def classify_batch_route():
+    """Запустить классификацию. ≤10 файлов — sync, >10 — async (Thread)."""
+    body = request.get_json()
+    batch_id = body.get("batch_id")
+    if not batch_id:
+        return jsonify(ok=False, error="batch_id required"), 400
+
+    # Guard: уже запущена?
+    if batch_id in _classify_locks:
+        t = _classify_locks[batch_id]
+        if t.is_alive():
+            return jsonify(ok=False, error="classify already running"), 409
+        else:
+            del _classify_locks[batch_id]
+
+    pending = documents.list_pending(batch_id)
+    total = len(pending)
+    if total == 0:
+        return jsonify(ok=False, error="no pending documents"), 400
+
+    # Sync для малых батчей
+    if total <= 10:
+        result = classify_batch(batch_id)
+        return jsonify(result)
+
+    # Async для больших
+    def _run():
+        try:
+            classify_batch(batch_id)
+        finally:
+            _classify_locks.pop(batch_id, None)
+
+    t = threading.Thread(target=_run, daemon=True)
+    _classify_locks[batch_id] = t
+    t.start()
+
+    return jsonify(ok=True, total=total), 202

site/routes/prompts_bp.py

@@ -0,0 +1,71 @@
+"""Prompts blueprint — CRUD + activate для версионирования промптов."""
+from flask import Blueprint, request, jsonify
+from site.db import prompts as db_prompts
+
+prompts_bp = Blueprint("prompts", __name__)
+
+
+@prompts_bp.route("/api/prompts", methods=["GET"])
+def prompts_get():
+    role = request.args.get("role")
+    if not role:
+        return jsonify(ok=False, error="role required"), 400
+    p = db_prompts.get_active(role)
+    if p:
+        return jsonify(ok=True, **{
+            "id": p["id"], "role": p["role"], "name": p["name"],
+            "body": p["body"], "is_active": p["is_active"],
+            "notes": p.get("notes"), "created_at": str(p.get("created_at", "")),
+        })
+    return jsonify(ok=False, error="not found"), 404
+
+
+@prompts_bp.route("/api/prompts/list", methods=["GET"])
+def prompts_list():
+    role = request.args.get("role")
+    if not role:
+        return jsonify(ok=False, error="role required"), 400
+    versions = db_prompts.list_by_role(role)
+    return jsonify(ok=True, versions=versions)
+
+
+@prompts_bp.route("/api/prompts/save", methods=["POST"])
+def prompts_save():
+    body = request.get_json()
+    if not body:
+        return jsonify(ok=False, error="body required"), 400
+    role = body.get("role", "")
+    name = body.get("name", "")
+    prompt_body = body.get("body", "")
+    notes = body.get("notes", "")
+    try:
+        p = db_prompts.insert(role, name, prompt_body, notes)
+        return jsonify(ok=True, id=p["id"])
+    except Exception as e:
+        return jsonify(ok=False, error=str(e)), 500
+
+
+@prompts_bp.route("/api/prompts/activate", methods=["POST"])
+def prompts_activate():
+    body = request.get_json()
+    if not body:
+        return jsonify(ok=False, error="body required"), 400
+    prompt_id = body.get("id")
+    try:
+        db_prompts.activate(prompt_id)
+        return jsonify(ok=True)
+    except Exception as e:
+        return jsonify(ok=False, error=str(e)), 500
+
+
+@prompts_bp.route("/api/prompts/delete", methods=["POST"])
+def prompts_delete():
+    body = request.get_json()
+    if not body:
+        return jsonify(ok=False, error="body required"), 400
+    prompt_id = body.get("id")
+    try:
+        db_prompts.delete(prompt_id)
+        return jsonify(ok=True)
+    except Exception as e:
+        return jsonify(ok=False, error=str(e)), 500

site/routes/upload_bp.py

@@ -0,0 +1,145 @@
+"""Upload blueprint — загрузка, конвертация, распаковка."""
+import io, os, base64, hashlib, zipfile, tempfile, subprocess
+from flask import Blueprint, request, jsonify, send_file
+from site.services.parse import parse_file
+from site.db import documents
+from site.config import MAX_CONTENT_LENGTH
+
+upload_bp = Blueprint("upload", __name__)
+
+ALLOWED = {"pdf", "docx", "doc", "zip"}
+
+
+def _check_ext(filename: str) -> str | None:
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    if ext not in ALLOWED:
+        return f"unsupported format: .{ext} (allowed: {', '.join(sorted(ALLOWED))})"
+    return None
+
+
+@upload_bp.route("/upload", methods=["POST"])
+def upload():
+    """Загрузка одного файла + авто-парсинг → БД."""
+    f = request.files.get("files")
+    if not f:
+        return jsonify(ok=False, error="no file"), 400
+
+    err = _check_ext(f.filename)
+    if err:
+        return jsonify(ok=False, error=err), 400
+
+    data = f.read()
+    content_hash = hashlib.sha256(data).hexdigest()[:16]
+    batch_id = request.form.get("batch_id")
+    zip_source = request.form.get("zip_source")
+
+    # Дедупликация по хешу
+    if batch_id:
+        existing = documents.get_by_hash(batch_id, content_hash)
+        if existing:
+            return jsonify(ok=False, error="duplicate", doc_id=existing["id"])
+
+    doc = documents.insert(
+        filename=f.filename,
+        mime_type=f.content_type or "application/octet-stream",
+        original_bytes=base64.b64encode(data).decode(),
+        batch_id=batch_id,
+        zip_source=zip_source,
+        content_hash=content_hash,
+    )
+
+    # Авто-парсинг
+    try:
+        result = parse_file(f.filename, data)
+        if result["status"] == "parsed":
+            documents.set_parsed(doc["id"], result["elements"])
+        else:
+            documents.set_error(doc["id"], result.get("error", "parse failed"))
+    except Exception as e:
+        documents.set_error(doc["id"], str(e))
+        result = {"status": "error", "error": str(e)}
+
+    contract_id = request.form.get("contract_id")
+    return jsonify(
+        ok=True,
+        doc_id=doc["id"],
+        contract_id=contract_id,
+        parsed={"status": result["status"], "element_count": result.get("element_count", 0)},
+    )
+
+
+@upload_bp.route("/convert-doc", methods=["POST"])
+def convert_doc():
+    """.doc → .docx через libreoffice (без сохранения на диск)."""
+    f = request.files.get("files")
+    if not f:
+        return jsonify(ok=False, error="no file"), 400
+
+    data = f.read()
+    doc_path = None
+    tmpdir = None
+    try:
+        with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as tmp:
+            tmp.write(data)
+            doc_path = tmp.name
+        tmpdir = tempfile.mkdtemp()
+        subprocess.run(
+            ["libreoffice", "--headless", "--convert-to", "docx", "--outdir", tmpdir, doc_path],
+            timeout=30, capture_output=True,
+        )
+        docx_files = [x for x in os.listdir(tmpdir) if x.endswith(".docx")]
+        if docx_files:
+            with open(os.path.join(tmpdir, docx_files[0]), "rb") as out:
+                return send_file(
+                    io.BytesIO(out.read()),
+                    mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                )
+        return jsonify(ok=False, error="conversion produced no output"), 500
+    finally:
+        if doc_path and os.path.exists(doc_path):
+            os.unlink(doc_path)
+        if tmpdir and os.path.exists(tmpdir):
+            for x in os.listdir(tmpdir):
+                os.unlink(os.path.join(tmpdir, x))
+            os.rmdir(tmpdir)
+
+
+@upload_bp.route("/unzip-upload", methods=["POST"])
+def unzip_upload():
+    """Распаковать ZIP → список файлов (base64 для фронтенда)."""
+    f = request.files.get("files")
+    if not f:
+        return jsonify(ok=False, error="no file"), 400
+
+    data = f.read()
+    MAX_FILES = 500
+    MAX_UNCOMPRESSED = 500 * 1024 * 1024  # 500 MB
+
+    files = []
+    total = 0
+
+    with zipfile.ZipFile(io.BytesIO(data)) as zf:
+        if len(zf.namelist()) > MAX_FILES:
+            return jsonify(ok=False, error=f"too many files in ZIP (max {MAX_FILES})"), 400
+
+        for info in zf.infolist():
+            if info.is_dir():
+                continue
+            name = os.path.basename(info.filename)
+            if not name or ".." in name or "/" in name or "\\" in name:
+                continue
+
+            raw = zf.read(info)
+            total += len(raw)
+            if total > MAX_UNCOMPRESSED:
+                return jsonify(ok=False, error="total uncompressed size exceeds 500 MB"), 400
+
+            ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
+            files.append({
+                "filename": name,
+                "ext": ext,
+                "size": len(raw),
+                "data_b64": base64.b64encode(raw).decode(),
+            })
+
+    return jsonify(ok=True, files=files)