v2.0.11: вся загрузка через ВМ-буфер (ZIP + .doc)

- /api/unzip_refs: pull ZIP с ВМ + распаковка
- /api/convert_refs: pull .doc с ВМ + конвертация в .docx
- convertDoc/addZipFile: PUT на ВМ вместо прямого multipart

site/config.py

@@ -1,7 +1,7 @@
 """Конфигурация приложения — все настройки в одном месте."""
 import os
 
-VERSION = "2.0.10"
+VERSION = "2.0.11"
 
 LLM_URL = os.getenv("LLM_API_URL", "https://api.aillm.ru/v1/chat/completions")
 LLM_KEY = os.getenv("LLM_API_KEY", "")

site/routes/upload_bp.py

@@ -50,6 +50,78 @@ def _delete_from_vm(url: str) -> None:
         pass
 
 
+def _pull_from_ref(ref):
+    """SSRF-проверка → лимит → pull с ретраями → DELETE с ВМ. Возвращает (name, content)."""
+    name = _safe_name(str(ref.get("name", "") or ""))
+    size = int(ref.get("size") or 0)
+    url = str(ref.get("url", "") or "")
+
+    if not url.startswith(config.VM_UPLOAD_PREFIX):
+        raise Exception("invalid url (SSRF guard)")
+    if size > config.VM_UPLOAD_MAX_BYTES:
+        _delete_from_vm(url)
+        raise Exception(f"file too large: {size} bytes (max {config.VM_UPLOAD_MAX_BYTES})")
+
+    content = _pull_with_retries(url)
+    if len(content) > config.VM_UPLOAD_MAX_BYTES:
+        _delete_from_vm(url)
+        raise Exception("file too large after pull")
+
+    _delete_from_vm(url)
+    return name, content
+
+
+def _unzip(data: bytes):
+    """Распаковать ZIP → (ok, files, error)."""
+    MAX_FILES = 500
+    MAX_UNCOMPRESSED = 500 * 1024 * 1024  # 500 MB
+    files = []
+    total = 0
+    try:
+        with zipfile.ZipFile(io.BytesIO(data)) as zf:
+            if len(zf.namelist()) > MAX_FILES:
+                return False, None, f"too many files in ZIP (max {MAX_FILES})"
+            for info in zf.infolist():
+                if info.is_dir():
+                    continue
+                name = os.path.basename(info.filename)
+                if not name or ".." in name or "/" in name or "\\" in name:
+                    continue
+                raw = zf.read(info)
+                total += len(raw)
+                if total > MAX_UNCOMPRESSED:
+                    return False, None, "total uncompressed size exceeds 500 MB"
+                ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
+                files.append({
+                    "filename": name,
+                    "ext": ext,
+                    "size": len(raw),
+                    "data_b64": base64.b64encode(raw).decode(),
+                })
+    except zipfile.BadZipFile:
+        return False, None, "invalid ZIP archive"
+    return True, files, None
+
+
+def _convert(filename: str, data: bytes) -> bytes:
+    """.doc → .docx через внешний libreoffice-сервис. Возвращает docx-байты."""
+    try:
+        resp = httpx.post(
+            config.CONVERT_SERVICE_URL + "/convert",
+            files={"file": (filename, data, "application/msword")},
+            timeout=120,
+        )
+    except httpx.TimeoutException:
+        raise Exception("conversion timeout")
+    if resp.status_code != 200:
+        try:
+            err = resp.json().get("error", "conversion failed")
+        except Exception:
+            err = "conversion failed"
+        raise Exception(err)
+    return resp.content
+
+
 def _store_and_parse(filename: str, data: bytes, batch_id, contract_id, zip_source=None, mime_type="application/octet-stream"):
     """Общая логика: дедуп → insert в БД → авто-парсинг. Возвращает dict-результат."""
     content_hash = hashlib.sha256(data).hexdigest()[:16]
@@ -125,34 +197,12 @@ def upload_refs():
 
     results = []
     for ref in files:
-        name = _safe_name(str(ref.get("name", "") or ""))
-        size = int(ref.get("size") or 0)
-        url = str(ref.get("url", "") or "")
-
-        # SSRF-защита: тянуть можно ТОЛЬКО с доверенного ВМ-буфера
-        if not url.startswith(config.VM_UPLOAD_PREFIX):
-            results.append({"name": name, "ok": False, "error": "invalid url (SSRF guard)"})
-            continue
-
-        # Лимит по заявленному размеру
-        if size > config.VM_UPLOAD_MAX_BYTES:
-            _delete_from_vm(url)
-            results.append({"name": name, "ok": False, "error": f"file too large: {size} bytes (max {config.VM_UPLOAD_MAX_BYTES})", "skipped": True})
-            continue
-
+        ref_name = _safe_name(str(ref.get("name", "") or ""))
         try:
-            content = _pull_with_retries(url)
+            name, content = _pull_from_ref(ref)
         except Exception as e:
-            results.append({"name": name, "ok": False, "error": f"pull failed: {e}"})
+            results.append({"name": ref_name, "ok": False, "error": str(e)})
             continue
-
-        # Реальная проверка размера после pull
-        if len(content) > config.VM_UPLOAD_MAX_BYTES:
-            _delete_from_vm(url)
-            results.append({"name": name, "ok": False, "error": "file too large after pull", "skipped": True})
-            continue
-
-        _delete_from_vm(url)
         stored = _store_and_parse(name, content, batch_id, contract_id, zip_source)
         results.append({"name": name, **stored})
 
@@ -161,66 +211,67 @@ def upload_refs():
 
 @upload_bp.route("/convert-doc", methods=["POST"])
 def convert_doc():
-    """.doc → .docx через внешний libreoffice-сервис (HTTP)."""
+    """.doc → .docx через внешний libreoffice-сервис (прямой multipart)."""
     f = request.files.get("files")
     if not f:
         return jsonify(ok=False, error="no file"), 400
-
     try:
-        resp = httpx.post(
-            config.CONVERT_SERVICE_URL + "/convert",
-            files={"file": (f.filename, f.read(), f.content_type or "application/msword")},
-            timeout=120,
-        )
-        if resp.status_code == 200:
-            return send_file(
-                io.BytesIO(resp.content),
-                mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            )
-        data = resp.json()
-        return jsonify(ok=False, error=data.get("error", "conversion failed")), 500
-    except httpx.TimeoutException:
-        return jsonify(ok=False, error="conversion timeout"), 500
+        content = _convert(f.filename, f.read())
     except Exception as e:
         return jsonify(ok=False, error=str(e)), 500
+    return send_file(
+        io.BytesIO(content),
+        mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    )
+
+
+@upload_bp.route("/api/convert_refs", methods=["POST"])
+def convert_refs():
+    """.doc → .docx через ВМ-буфер (паттерн drhider): pull .doc с ВМ → конвертация → docx."""
+    data = request.get_json(silent=True) or {}
+    files = data.get("files") or []
+    if not files:
+        return jsonify(ok=False, error="no files"), 400
+    ref = files[0]
+    try:
+        name, content = _pull_from_ref(ref)
+    except Exception as e:
+        return jsonify(ok=False, error=str(e)), 400
+    try:
+        docx = _convert(name, content)
+    except Exception as e:
+        return jsonify(ok=False, error=str(e)), 500
+    return send_file(
+        io.BytesIO(docx),
+        mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    )
 
 
 @upload_bp.route("/unzip-upload", methods=["POST"])
 def unzip_upload():
-    """Распаковать ZIP → список файлов (base64 для фронтенда)."""
+    """Распаковать ZIP → список файлов (base64 для фронтенда, прямой multipart)."""
     f = request.files.get("files")
     if not f:
         return jsonify(ok=False, error="no file"), 400
-
-    data = f.read()
-    MAX_FILES = 500
-    MAX_UNCOMPRESSED = 500 * 1024 * 1024  # 500 MB
-
-    files = []
-    total = 0
-
-    with zipfile.ZipFile(io.BytesIO(data)) as zf:
-        if len(zf.namelist()) > MAX_FILES:
-            return jsonify(ok=False, error=f"too many files in ZIP (max {MAX_FILES})"), 400
-
-        for info in zf.infolist():
-            if info.is_dir():
-                continue
-            name = os.path.basename(info.filename)
-            if not name or ".." in name or "/" in name or "\\" in name:
-                continue
-
-            raw = zf.read(info)
-            total += len(raw)
-            if total > MAX_UNCOMPRESSED:
-                return jsonify(ok=False, error="total uncompressed size exceeds 500 MB"), 400
-
-            ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
-            files.append({
-                "filename": name,
-                "ext": ext,
-                "size": len(raw),
-                "data_b64": base64.b64encode(raw).decode(),
-            })
-
+    ok, files, err = _unzip(f.read())
+    if not ok:
+        return jsonify(ok=False, error=err), 400
     return jsonify(ok=True, files=files)
+
+
+@upload_bp.route("/api/unzip_refs", methods=["POST"])
+def unzip_refs():
+    """Распаковать ZIP через ВМ-буфер (паттерн drhider): pull ZIP с ВМ → распаковка."""
+    data = request.get_json(silent=True) or {}
+    files = data.get("files") or []
+    if not files:
+        return jsonify(ok=False, error="no files"), 400
+    ref = files[0]
+    try:
+        name, content = _pull_from_ref(ref)
+    except Exception as e:
+        return jsonify(ok=False, error=str(e)), 400
+    ok, unzipped, err = _unzip(content)
+    if not ok:
+        return jsonify(ok=False, error=err), 400
+    return jsonify(ok=True, files=unzipped)

site/static/files.js

@@ -207,21 +207,29 @@ window.toggleClassifyDetail = async function(i) {
 };
 
 /**
- * convertDoc(file, onProgress) — .doc → .docx через внешний libreoffice-сервис.
- * С честным счётчиком времени.
+ * convertDoc(file, onProgress) — .doc → .docx через ВМ-буфер (паттерн drhider).
+ * Фаза 1: PUT .doc на ВМ. Фаза 2: /api/convert_refs → pull → конвертация → .docx.
  */
 function convertDoc(file, onProgress) {
   var startTime = Date.now();
   if (onProgress) onProgress({ kind: 'converting', elapsed: 0 });
-  var fd = new FormData();
-  fd.append('files', file, file.name);
+  var token = crypto.randomUUID();
+  var vmUrl = VM_UPLOAD_URL + token + '_0';
 
   var timer = setInterval(function() {
     var elapsed = Math.floor((Date.now() - startTime) / 1000);
     if (onProgress) onProgress({ kind: 'converting', elapsed: elapsed });
   }, 1000);
 
-  return fetch(CONVERT_URL + '?_=' + Date.now(), { method: 'POST', body: fd })
+  return fetch(vmUrl, { method: 'PUT', body: file, headers: { 'Content-Type': 'application/octet-stream' } })
+    .then(function(r) {
+      if (!r.ok) throw new Error('Конвертация: VM upload HTTP ' + r.status);
+      return fetch('/api/convert_refs?_=' + Date.now(), {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ files: [{ name: file.name, size: file.size, url: vmUrl }] })
+      });
+    })
     .then(function(r) {
       if (!r.ok) throw new Error('Конвертация: HTTP ' + r.status);
       return r.blob();
@@ -236,6 +244,7 @@ function convertDoc(file, onProgress) {
     })
     .catch(function(e) {
       clearInterval(timer);
+      if (e.message === 'Failed to fetch' || e.name === 'TypeError') throw new Error('Конвертация: Сеть');
       throw e;
     });
 }
@@ -386,19 +395,18 @@ async function addZipFile(file) {
   render(state);
 
   try {
-    // Шаг 1: распаковать ZIP на бэкенде
-    var zipResp = await new Promise(function(resolve, reject) {
-      var xhr = new XMLHttpRequest();
-      xhr.open('POST', UNZIP_URL);
-      xhr.responseType = 'json';
-      xhr.onload = function() { resolve(xhr.response); };
-      xhr.onerror = function() { reject(new Error('Сеть')); };
-      xhr.ontimeout = function() { reject(new Error('Таймаут')); };
-      xhr.timeout = 60000;
-      var fd = new FormData();
-      fd.append('files', file);
-      xhr.send(fd);
+    // Шаг 1: распаковать ZIP через ВМ-буфер (паттерн drhider)
+    var token = crypto.randomUUID();
+    var vmUrl = VM_UPLOAD_URL + token + '_0';
+    var putResp = await fetch(vmUrl, { method: 'PUT', body: file, headers: { 'Content-Type': 'application/octet-stream' } });
+    if (!putResp.ok) throw new Error('unzip failed: VM upload HTTP ' + putResp.status);
+    var refsResp = await fetch('/api/unzip_refs?_=' + Date.now(), {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ files: [{ name: file.name, size: file.size, url: vmUrl }] })
     });
+    if (!refsResp.ok) throw new Error('unzip failed: HTTP ' + refsResp.status);
+    var zipResp = await refsResp.json();
     if (!zipResp.ok || !zipResp.files) throw new Error('unzip failed');
 
     // Подтверждение: показать первые 10 файлов + итог

site/templates/index.html

@@ -73,7 +73,7 @@
 <body>
 <div class="topbar">
   <img src="/static/logo.svg" alt="Nubes">
-  <span class="title">Сверка договоров — LLM AI-driven Event Sourcing <span style="font-weight:400;color:var(--muted);font-size:12px;">v2.0.10</span></span>
+  <span class="title">Сверка договоров — LLM AI-driven Event Sourcing <span style="font-weight:400;color:var(--muted);font-size:12px;">v2.0.11</span></span>
   <div id="pipelineStepper" style="display:flex;gap:8px;font-size:11px;align-items:center;color:var(--muted);">
     <span id="stepUpload">○ Загрузка</span><span>→</span>
     <span id="stepClassify">○ Классификация</span><span>→</span>
@@ -218,7 +218,7 @@
 
 <script src="/static/state.js?v=2.0.8"></script>
 <script src="/static/app_utils.js?v=2.0.8"></script>
-<script src="/static/files.js?v=2.0.10"></script>
+<script src="/static/files.js?v=2.0.11"></script>
 <script src="/static/groups.js?v=2.0.8"></script>
 <script src="/static/compare.js?v=2.0.8"></script>
 <script src="/static/app.js?v=2.0.10"></script>