fix: двухшаговый процесс — загрузка (быстро) → обработка (LLM)

Шаг 1: POST / → сохранить файлы, парсинг → редирект на /?id=X Шаг 2: POST /process/X → LLM extract + diff → редирект на /?id=X Нет таймаута — загрузка мгновенная, LLM по отдельной кнопке.
2026-06-15 07:41:13 +04:00
parent 105459ef1a
commit fdd7d89ce5
2 changed files with 111 additions and 61 deletions
@@ -1,7 +1,7 @@
-"""app.py — Точка входа и сборка слоёв. Бизнес-логики НОЛЬ."""
+"""app.py — Точка входа и сборка слоёв."""

 from dotenv import load_dotenv
-from flask import Flask, render_template, request
+from flask import Flask, render_template, request, redirect
 import json

 import schema
@@ -25,7 +25,8 @@ class ContractsApp:
        self._add_routes()

    def _add_routes(self):
-        self.app.add_url_rule("/", "index", self._upload_page, methods=["GET", "POST"])
+        self.app.add_url_rule("/", "index", self._index, methods=["GET", "POST"])
+        self.app.add_url_rule("/process/<cid>", "process", self._process, methods=["POST"])
        self.app.add_url_rule("/health", "health", self._health)
        self.app.register_blueprint(test_bp)
        self.app.register_blueprint(upload_bp)
@@ -34,17 +35,22 @@ class ContractsApp:
    def _health(self):
        return "OK", 200, {"Content-Type": "text/plain"}

-    def _upload_page(self):
+    # ── Шаг 1: загрузка файлов ──────────────────────────────────
+
+    def _index(self):
        if request.method == "POST":
-            return self._process_upload()
+            return self._upload_files()
+        cid = request.args.get("id")
+        if cid:
+            return self._show_contract(cid)
        return render_template("upload.html")

-    def _process_upload(self):
+    def _upload_files(self):
+        """Только сохранить файлы, без LLM."""
        files = request.files.getlist("files")
        if not files or not any(f.filename for f in files):
            return render_template("upload.html", error="Нет файлов")

-        # Создать договор
        conn, err = db.connect()
        if err:
            return render_template("upload.html", error=f"БД: {err}")
@@ -58,9 +64,6 @@ class ContractsApp:
        cur.close()
        conn.close()

-        results = []
-        total_rows = 0
-
        for f in files:
            if not f.filename:
                continue
@@ -68,7 +71,6 @@ class ContractsApp:
            mime = mimeutil.guess_mime(filename) or f.content_type or "application/octet-stream"
            file_bytes = f.read()

-            # Сохранить документ
            db.execute(
                "INSERT INTO documents (filename, mime_type, original_bytes, status) VALUES (%s,%s,%s,'uploaded')",
                (filename, mime, file_bytes),
@@ -79,72 +81,105 @@ class ContractsApp:
            )
            doc_id = doc["id"] if doc else None
            if not doc_id:
-                results.append({"name": filename, "ok": False, "error": "doc save failed"})
                continue

-            # Парсинг
+            # Парсинг (быстро, без LLM)
            pr = parser_mod.parse(file_bytes, mime)
-
-            # ZIP: объединяем элементы из всех файлов
+            elements = []
            if mime == "application/zip" and "files" in pr:
-                all_el = []
-                zip_errs = []
                for zf in pr["files"]:
-                    if zf.get("errors"):
-                        zip_errs.extend(zf["errors"])
-                    all_el.extend(zf.get("elements", []))
-                if zip_errs and not all_el:
-                    results.append({"name": filename, "ok": False, "error": "; ".join(zip_errs)})
-                    continue
-                elements = all_el
+                    elements.extend(zf.get("elements", []))
            else:
                elements = pr.get("elements", [])

-            if not elements:
-                errs = pr.get("errors", [])
-                results.append({"name": filename, "ok": False, "error": "; ".join(errs) if errs else "нет таблиц"})
-                continue
-
-            text = textify.to_text(elements)
+            text = textify.to_text(elements) if elements else ""
            db.execute("UPDATE documents SET parsed_text=%s, status='parsed' WHERE id=%s", (text, str(doc_id)))

-            # Допник
            db.execute(
                "INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')",
                (str(contract_id), str(doc_id)),
            )
-            supp, _ = db.query_one(
-                "SELECT id FROM supplements WHERE document_id=%s ORDER BY created_at DESC LIMIT 1",
-                (str(doc_id),),
-            )
-            supp_id = supp["id"] if supp else None
-            if not supp_id:
-                results.append({"name": filename, "ok": False, "error": "supplement save failed"})
+
+        return redirect("/?id=" + str(contract_id))
+
+    def _show_contract(self, cid):
+        """Показать договор с кнопкой «Обработать»."""
+        contract, _ = db.query_one("SELECT id,number,created_at FROM contracts WHERE id=%s", (cid,))
+        if not contract:
+            return render_template("upload.html", error="Договор не найден")
+
+        supps, _ = db.query(
+            "SELECT s.id, s.created_at, d.filename, d.status, d.parsed_text IS NOT NULL as has_text "
+            "FROM supplements s JOIN documents d ON s.document_id=d.id "
+            "WHERE s.contract_id=%s ORDER BY s.created_at",
+            (cid,),
+        )
+        supp_list = [dict(zip(supps["columns"], r)) for r in supps["rows"]] if supps else []
+
+        # Есть ли уже результаты?
+        rows_res, _ = db.query(
+            "SELECT sr.row_num,sr.name,sr.price,sr.qty,sr.sum,sr.date_start "
+            "FROM spec_rows sr JOIN supplements s ON sr.supplement_id=s.id "
+            "WHERE s.contract_id=%s ORDER BY sr.row_num",
+            (cid,),
+        )
+        all_rows = [dict(zip(rows_res["columns"], r)) for r in rows_res["rows"]] if rows_res else []
+
+        # Есть необработанные допники?
+        unprocessed = [s for s in supp_list if s["status"] in ("uploaded", "parsed")]
+
+        return render_template("upload.html",
+            contract=contract, supplements=supp_list,
+            all_rows=all_rows, unprocessed=unprocessed)
+
+    # ── Шаг 2: LLM-обработка ────────────────────────────────────
+
+    def _process(self, cid):
+        """Запустить LLM-обработку для всех необработанных допников договора."""
+        supps, _ = db.query(
+            "SELECT s.id, d.id as doc_id, d.parsed_text, d.filename "
+            "FROM supplements s JOIN documents d ON s.document_id=d.id "
+            "WHERE s.contract_id=%s AND d.parsed_text IS NOT NULL",
+            (cid,),
+        )
+        if not supps:
+            return redirect("/?id=" + cid)
+
+        supp_rows = [dict(zip(supps["columns"], r)) for r in supps["rows"]]
+        total_rows = 0
+
+        for s in supp_rows:
+            text = s["parsed_text"]
+            if not text:
+                continue
+
+            # Проверим, не обработан ли уже
+            existing, _ = db.query("SELECT 1 FROM spec_rows WHERE supplement_id=%s LIMIT 1", (s["id"],))
+            if existing and existing["rows"]:
                continue

-            # LLM-извлечение
            ext = extractor.extract(text)
            if "error" in ext:
-                results.append({"name": filename, "ok": False, "error": ext["error"]})
+                db.execute("UPDATE documents SET status='extract_error', error_message=%s WHERE id=%s",
+                           (ext["error"], s["doc_id"]))
                continue

            rows = ext.get("rows", [])
            total_rows += len(rows)

-            # Сохранить spec_rows
            for row in rows:
                db.execute(
                    "INSERT INTO spec_rows (supplement_id,row_num,name,price,qty,sum,date_start) VALUES (%s,%s,%s,%s,%s,%s,%s)",
-                    (str(supp_id), row.get("row_num"), row.get("name"),
+                    (s["id"], row.get("row_num"), row.get("name"),
                     row.get("price"), row.get("qty"), row.get("sum"), row.get("date_start")),
                )

-            # Diff с предыдущим
+            # Diff
            prev_res, _ = db.query(
                "SELECT sr.row_num,sr.name,sr.price,sr.qty,sr.sum,sr.date_start "
-                "FROM spec_rows sr JOIN supplements s ON sr.supplement_id=s.id "
-                "WHERE s.contract_id=%s AND s.id!=%s ORDER BY sr.row_num",
-                (str(contract_id), str(supp_id)),
+                "FROM spec_rows sr JOIN supplements sup ON sr.supplement_id=sup.id "
+                "WHERE sup.contract_id=%s AND sup.id!=%s ORDER BY sr.row_num",
+                (cid, s["id"]),
            )
            prev_rows = [dict(zip(prev_res["columns"], r)) for r in prev_res["rows"]] if prev_res else []
            diff_r = differ.diff(prev_rows, rows)
@@ -153,26 +188,14 @@ class ContractsApp:
                db.execute(
                    "INSERT INTO spec_history (contract_id,supplement_id,row_num,change_type,old_values,new_values) "
                    "VALUES (%s,%s,%s,%s,%s,%s)",
-                    (str(contract_id), str(supp_id), ch["row_num"], ch["change_type"],
+                    (cid, s["id"], ch["row_num"], ch["change_type"],
                     json.dumps(ch.get("old_values"), ensure_ascii=False, default=str) if ch.get("old_values") else None,
                     json.dumps(ch.get("new_values"), ensure_ascii=False, default=str) if ch.get("new_values") else None),
                )

-            db.execute("UPDATE documents SET status='extracted' WHERE id=%s", (str(doc_id),))
-            results.append({"name": filename, "ok": True, "rows": len(rows), "summary": diff_r.get("summary", {})})
+            db.execute("UPDATE documents SET status='extracted' WHERE id=%s", (s["doc_id"],))

-        # Все строки для отображения
-        all_rows = []
-        rows_res, _ = db.query(
-            "SELECT sr.row_num,sr.name,sr.price,sr.qty,sr.sum,sr.date_start "
-            "FROM spec_rows sr JOIN supplements s ON sr.supplement_id=s.id "
-            "WHERE s.contract_id=%s ORDER BY sr.row_num",
-            (str(contract_id),),
-        )
-        if rows_res:
-            all_rows = [dict(zip(rows_res["columns"], r)) for r in rows_res["rows"]]
-
-        return render_template("upload.html", results=results, all_rows=all_rows, total_rows=total_rows)
+        return redirect("/?id=" + cid)

    def run(self):
        self.app.run(host="0.0.0.0", port=5000)
@@ -106,6 +106,33 @@
      </div>
    </div>

+    <!-- Загруженные допники + кнопка Обработать -->
+    {% if contract %}
+    <div class="card">
+      <div class="card-header">
+        <i data-lucide="file-check" style="width:18px;height:18px;"></i>
+        Договор {{ contract.number }} — файлы загружены
+      </div>
+      <div class="card-body">
+        {% for s in supplements %}
+        <div class="queue-item">
+          <span class="badge {{ 'badge-ok' if s.status == 'extracted' else 'badge-err' }}">{{ '✅' if s.status == 'extracted' else '⏳' }}</span>
+          <span class="name">{{ s.filename }}</span>
+          <span class="size">{{ s.status }}</span>
+        </div>
+        {% endfor %}
+
+        {% if unprocessed %}
+        <form method="POST" action="/process/{{ contract.id }}" style="margin-top:12px;">
+          <button type="submit" class="btn btn-primary" style="width:100%;justify-content:center;">
+            <i data-lucide="play" style="width:16px;height:16px;"></i> Обработать (LLM)
+          </button>
+        </form>
+        {% endif %}
+      </div>
+    </div>
+    {% endif %}
+
    <!-- Результаты (после обработки) -->
    {% if results %}
    <div class="card">