Improve LLM and CSV processing

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

site/routes/api_bp.py

@@ -17,12 +17,13 @@ import queue
 import threading
 import time
 import zipfile
-import traceback
 import logging
 from datetime import datetime, timedelta
-from flask import Blueprint, request, send_file, jsonify, Response, stream_with_context
+from flask import (Blueprint, current_app, request, send_file, jsonify, Response,
+                   stream_with_context)
 
 from drhider import obfuscate_files, LLMClient
+from drhider.obfuscator import LLMIncompleteError
 from upload.backend.session import (create_session, add_file, get_files, store_result,
                                     get_result, store_csv, get_csv, cleanup, file_count,
                                     MAX_FILE_BYTES, pause_ttl, resume_ttl,
@@ -78,6 +79,8 @@ def upload():
 @api_bp.route("/session_files/<sid>", methods=["GET"])
 def session_files(sid):
     """Отладка: список файлов сессии с размерами (для теста загрузки)."""
+    if not current_app.config.get("ENABLE_SESSION_FILES"):
+        return jsonify({"ok": False, "error": "Not found"}), 404
     files = get_files(sid)
     if files is None:
         return jsonify({"ok": False, "error": "Session not found"}), 404
@@ -118,6 +121,9 @@ def process_stream(sid):
 
     all_files = [(fname, content, "") for fname, content in files]
     log.info("process_stream: start sid=%s files=%d", sid, len(all_files))
+    # A failed rerun must not leave an older ZIP or mapping downloadable.
+    store_result(sid, None)
+    store_csv(sid, None)
     # Сессия живёт, пока идёт обработка (TTL возобновляется в finally генератора)
     pause_ttl(sid)
 
@@ -172,9 +178,16 @@ def process_stream(sid):
                     log.info("worker: done sid=%s in %.1fs tokens=%d llm_sec=%.1f zip_len=%d",
                              sid, dt, llm.tokens_total, llm.llm_sec, len(zip_data))
                     q.put(("result", zip_data, csv_str, stats))
-            except Exception as e:
-                log.error("worker: exception sid=%s: %r\n%s", sid, e, traceback.format_exc())
-                q.put(("error", repr(e)))
+            except LLMIncompleteError as e:
+                log.exception("worker: LLM incomplete sid=%s chunks=%d", sid, e.llm_errors)
+                q.put(("error", {
+                    "error": "Processing stopped because the LLM could not process all fragments",
+                    "llm_incomplete": True,
+                    "llm_errors": e.llm_errors,
+                }))
+            except Exception:
+                log.exception("worker: processing failed sid=%s", sid)
+                q.put(("error", {"error": "Processing failed"}))
 
         threading.Thread(target=worker, daemon=True).start()
         log.debug("process_stream: worker thread started sid=%s", sid)
@@ -266,7 +279,7 @@ def process_stream(sid):
                         store_csv(sid, csv_str)
                     count = 0
                     with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
-                        count = len([n for n in zf.namelist() if n != "mapping.csv"])
+                        count = len(zf.namelist())
                     log.info("process_stream: complete sid=%s count=%d stats=%r", sid, count, stats)
                     try:
                         yield (
@@ -295,10 +308,11 @@ def process_stream(sid):
                     return
 
                 elif kind == "error":
-                    _, msg = evt
-                    log.error("process_stream: error event sid=%s msg=%r", sid, msg)
+                    _, error_data = evt
+                    log.error("process_stream: error event sid=%s llm_incomplete=%s",
+                              sid, error_data.get("llm_incomplete", False))
                     try:
-                        yield f"event: proc_error\ndata: {json.dumps({'error': msg})}\n\n"
+                        yield f"event: proc_error\ndata: {json.dumps(error_data)}\n\n"
                     except _disconnect_exceptions() as e:
                         log.warning("process_stream: disconnect on error sid=%s err=%r", sid, e)
                         return
@@ -322,6 +336,8 @@ def process(sid):
         return jsonify({"ok": False, "error": "Session not found"}), 404
     if not files:
         return jsonify({"ok": False, "error": "No files"}), 400
+    store_result(sid, None)
+    store_csv(sid, None)
     try:
         llm = LLMClient()
         all_files = [(fname, content, "") for fname, content in files]
@@ -334,9 +350,17 @@ def process(sid):
         if csv_str:
             store_csv(sid, csv_str)
         return jsonify({"ok": True, "status": "done"})
-    except Exception as e:
-        traceback.print_exc()
-        return jsonify({"ok": False, "error": str(e)}), 500
+    except LLMIncompleteError as e:
+        log.exception("process: LLM incomplete sid=%s chunks=%d", sid, e.llm_errors)
+        return jsonify({
+            "ok": False,
+            "error": "Processing stopped because the LLM could not process all fragments",
+            "llm_incomplete": True,
+            "llm_errors": e.llm_errors,
+        }), 500
+    except Exception:
+        log.exception("process: processing failed sid=%s", sid)
+        return jsonify({"ok": False, "error": "Processing failed"}), 500
 
 
 @api_bp.route("/download/<sid>", methods=["GET"])