From 55bd7a027d6d903e5f556a054c03b6c95ffa5a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Mon, 24 Aug 2026 22:08:01 +0300 Subject: [PATCH] =?UTF-8?q?v2.0.11:=20=D0=B2=D1=81=D1=8F=20=D0=B7=D0=B0?= =?UTF-8?q?=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=B0=20=D1=87=D0=B5=D1=80=D0=B5?= =?UTF-8?q?=D0=B7=20=D0=92=D0=9C-=D0=B1=D1=83=D1=84=D0=B5=D1=80=20(ZIP=20+?= =?UTF-8?q?=20.doc)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /api/unzip_refs: pull ZIP с ВМ + распаковка - /api/convert_refs: pull .doc с ВМ + конвертация в .docx - convertDoc/addZipFile: PUT на ВМ вместо прямого multipart --- site/config.py | 2 +- site/routes/upload_bp.py | 199 ++++++++++++++++++++++++-------------- site/static/files.js | 42 ++++---- site/templates/index.html | 4 +- 4 files changed, 153 insertions(+), 94 deletions(-) diff --git a/site/config.py b/site/config.py index 38f1573..d51f0ac 100644 --- a/site/config.py +++ b/site/config.py @@ -1,7 +1,7 @@ """Конфигурация приложения — все настройки в одном месте.""" import os -VERSION = "2.0.10" +VERSION = "2.0.11" LLM_URL = os.getenv("LLM_API_URL", "https://api.aillm.ru/v1/chat/completions") LLM_KEY = os.getenv("LLM_API_KEY", "") diff --git a/site/routes/upload_bp.py b/site/routes/upload_bp.py index 3249ebb..e40abfc 100644 --- a/site/routes/upload_bp.py +++ b/site/routes/upload_bp.py @@ -50,6 +50,78 @@ def _delete_from_vm(url: str) -> None: pass +def _pull_from_ref(ref): + """SSRF-проверка → лимит → pull с ретраями → DELETE с ВМ. Возвращает (name, content).""" + name = _safe_name(str(ref.get("name", "") or "")) + size = int(ref.get("size") or 0) + url = str(ref.get("url", "") or "") + + if not url.startswith(config.VM_UPLOAD_PREFIX): + raise Exception("invalid url (SSRF guard)") + if size > config.VM_UPLOAD_MAX_BYTES: + _delete_from_vm(url) + raise Exception(f"file too large: {size} bytes (max {config.VM_UPLOAD_MAX_BYTES})") + + content = _pull_with_retries(url) + if len(content) > config.VM_UPLOAD_MAX_BYTES: + _delete_from_vm(url) + raise Exception("file too large after pull") + + _delete_from_vm(url) + return name, content + + +def _unzip(data: bytes): + """Распаковать ZIP → (ok, files, error).""" + MAX_FILES = 500 + MAX_UNCOMPRESSED = 500 * 1024 * 1024 # 500 MB + files = [] + total = 0 + try: + with zipfile.ZipFile(io.BytesIO(data)) as zf: + if len(zf.namelist()) > MAX_FILES: + return False, None, f"too many files in ZIP (max {MAX_FILES})" + for info in zf.infolist(): + if info.is_dir(): + continue + name = os.path.basename(info.filename) + if not name or ".." in name or "/" in name or "\\" in name: + continue + raw = zf.read(info) + total += len(raw) + if total > MAX_UNCOMPRESSED: + return False, None, "total uncompressed size exceeds 500 MB" + ext = name.rsplit(".", 1)[-1].lower() if "." in name else "" + files.append({ + "filename": name, + "ext": ext, + "size": len(raw), + "data_b64": base64.b64encode(raw).decode(), + }) + except zipfile.BadZipFile: + return False, None, "invalid ZIP archive" + return True, files, None + + +def _convert(filename: str, data: bytes) -> bytes: + """.doc → .docx через внешний libreoffice-сервис. Возвращает docx-байты.""" + try: + resp = httpx.post( + config.CONVERT_SERVICE_URL + "/convert", + files={"file": (filename, data, "application/msword")}, + timeout=120, + ) + except httpx.TimeoutException: + raise Exception("conversion timeout") + if resp.status_code != 200: + try: + err = resp.json().get("error", "conversion failed") + except Exception: + err = "conversion failed" + raise Exception(err) + return resp.content + + def _store_and_parse(filename: str, data: bytes, batch_id, contract_id, zip_source=None, mime_type="application/octet-stream"): """Общая логика: дедуп → insert в БД → авто-парсинг. Возвращает dict-результат.""" content_hash = hashlib.sha256(data).hexdigest()[:16] @@ -125,34 +197,12 @@ def upload_refs(): results = [] for ref in files: - name = _safe_name(str(ref.get("name", "") or "")) - size = int(ref.get("size") or 0) - url = str(ref.get("url", "") or "") - - # SSRF-защита: тянуть можно ТОЛЬКО с доверенного ВМ-буфера - if not url.startswith(config.VM_UPLOAD_PREFIX): - results.append({"name": name, "ok": False, "error": "invalid url (SSRF guard)"}) - continue - - # Лимит по заявленному размеру - if size > config.VM_UPLOAD_MAX_BYTES: - _delete_from_vm(url) - results.append({"name": name, "ok": False, "error": f"file too large: {size} bytes (max {config.VM_UPLOAD_MAX_BYTES})", "skipped": True}) - continue - + ref_name = _safe_name(str(ref.get("name", "") or "")) try: - content = _pull_with_retries(url) + name, content = _pull_from_ref(ref) except Exception as e: - results.append({"name": name, "ok": False, "error": f"pull failed: {e}"}) + results.append({"name": ref_name, "ok": False, "error": str(e)}) continue - - # Реальная проверка размера после pull - if len(content) > config.VM_UPLOAD_MAX_BYTES: - _delete_from_vm(url) - results.append({"name": name, "ok": False, "error": "file too large after pull", "skipped": True}) - continue - - _delete_from_vm(url) stored = _store_and_parse(name, content, batch_id, contract_id, zip_source) results.append({"name": name, **stored}) @@ -161,66 +211,67 @@ def upload_refs(): @upload_bp.route("/convert-doc", methods=["POST"]) def convert_doc(): - """.doc → .docx через внешний libreoffice-сервис (HTTP).""" + """.doc → .docx через внешний libreoffice-сервис (прямой multipart).""" f = request.files.get("files") if not f: return jsonify(ok=False, error="no file"), 400 - try: - resp = httpx.post( - config.CONVERT_SERVICE_URL + "/convert", - files={"file": (f.filename, f.read(), f.content_type or "application/msword")}, - timeout=120, - ) - if resp.status_code == 200: - return send_file( - io.BytesIO(resp.content), - mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ) - data = resp.json() - return jsonify(ok=False, error=data.get("error", "conversion failed")), 500 - except httpx.TimeoutException: - return jsonify(ok=False, error="conversion timeout"), 500 + content = _convert(f.filename, f.read()) except Exception as e: return jsonify(ok=False, error=str(e)), 500 + return send_file( + io.BytesIO(content), + mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ) + + +@upload_bp.route("/api/convert_refs", methods=["POST"]) +def convert_refs(): + """.doc → .docx через ВМ-буфер (паттерн drhider): pull .doc с ВМ → конвертация → docx.""" + data = request.get_json(silent=True) or {} + files = data.get("files") or [] + if not files: + return jsonify(ok=False, error="no files"), 400 + ref = files[0] + try: + name, content = _pull_from_ref(ref) + except Exception as e: + return jsonify(ok=False, error=str(e)), 400 + try: + docx = _convert(name, content) + except Exception as e: + return jsonify(ok=False, error=str(e)), 500 + return send_file( + io.BytesIO(docx), + mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ) @upload_bp.route("/unzip-upload", methods=["POST"]) def unzip_upload(): - """Распаковать ZIP → список файлов (base64 для фронтенда).""" + """Распаковать ZIP → список файлов (base64 для фронтенда, прямой multipart).""" f = request.files.get("files") if not f: return jsonify(ok=False, error="no file"), 400 - - data = f.read() - MAX_FILES = 500 - MAX_UNCOMPRESSED = 500 * 1024 * 1024 # 500 MB - - files = [] - total = 0 - - with zipfile.ZipFile(io.BytesIO(data)) as zf: - if len(zf.namelist()) > MAX_FILES: - return jsonify(ok=False, error=f"too many files in ZIP (max {MAX_FILES})"), 400 - - for info in zf.infolist(): - if info.is_dir(): - continue - name = os.path.basename(info.filename) - if not name or ".." in name or "/" in name or "\\" in name: - continue - - raw = zf.read(info) - total += len(raw) - if total > MAX_UNCOMPRESSED: - return jsonify(ok=False, error="total uncompressed size exceeds 500 MB"), 400 - - ext = name.rsplit(".", 1)[-1].lower() if "." in name else "" - files.append({ - "filename": name, - "ext": ext, - "size": len(raw), - "data_b64": base64.b64encode(raw).decode(), - }) - + ok, files, err = _unzip(f.read()) + if not ok: + return jsonify(ok=False, error=err), 400 return jsonify(ok=True, files=files) + + +@upload_bp.route("/api/unzip_refs", methods=["POST"]) +def unzip_refs(): + """Распаковать ZIP через ВМ-буфер (паттерн drhider): pull ZIP с ВМ → распаковка.""" + data = request.get_json(silent=True) or {} + files = data.get("files") or [] + if not files: + return jsonify(ok=False, error="no files"), 400 + ref = files[0] + try: + name, content = _pull_from_ref(ref) + except Exception as e: + return jsonify(ok=False, error=str(e)), 400 + ok, unzipped, err = _unzip(content) + if not ok: + return jsonify(ok=False, error=err), 400 + return jsonify(ok=True, files=unzipped) diff --git a/site/static/files.js b/site/static/files.js index 6f6bea6..a900588 100644 --- a/site/static/files.js +++ b/site/static/files.js @@ -207,21 +207,29 @@ window.toggleClassifyDetail = async function(i) { }; /** - * convertDoc(file, onProgress) — .doc → .docx через внешний libreoffice-сервис. - * С честным счётчиком времени. + * convertDoc(file, onProgress) — .doc → .docx через ВМ-буфер (паттерн drhider). + * Фаза 1: PUT .doc на ВМ. Фаза 2: /api/convert_refs → pull → конвертация → .docx. */ function convertDoc(file, onProgress) { var startTime = Date.now(); if (onProgress) onProgress({ kind: 'converting', elapsed: 0 }); - var fd = new FormData(); - fd.append('files', file, file.name); + var token = crypto.randomUUID(); + var vmUrl = VM_UPLOAD_URL + token + '_0'; var timer = setInterval(function() { var elapsed = Math.floor((Date.now() - startTime) / 1000); if (onProgress) onProgress({ kind: 'converting', elapsed: elapsed }); }, 1000); - return fetch(CONVERT_URL + '?_=' + Date.now(), { method: 'POST', body: fd }) + return fetch(vmUrl, { method: 'PUT', body: file, headers: { 'Content-Type': 'application/octet-stream' } }) + .then(function(r) { + if (!r.ok) throw new Error('Конвертация: VM upload HTTP ' + r.status); + return fetch('/api/convert_refs?_=' + Date.now(), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ files: [{ name: file.name, size: file.size, url: vmUrl }] }) + }); + }) .then(function(r) { if (!r.ok) throw new Error('Конвертация: HTTP ' + r.status); return r.blob(); @@ -236,6 +244,7 @@ function convertDoc(file, onProgress) { }) .catch(function(e) { clearInterval(timer); + if (e.message === 'Failed to fetch' || e.name === 'TypeError') throw new Error('Конвертация: Сеть'); throw e; }); } @@ -386,19 +395,18 @@ async function addZipFile(file) { render(state); try { - // Шаг 1: распаковать ZIP на бэкенде - var zipResp = await new Promise(function(resolve, reject) { - var xhr = new XMLHttpRequest(); - xhr.open('POST', UNZIP_URL); - xhr.responseType = 'json'; - xhr.onload = function() { resolve(xhr.response); }; - xhr.onerror = function() { reject(new Error('Сеть')); }; - xhr.ontimeout = function() { reject(new Error('Таймаут')); }; - xhr.timeout = 60000; - var fd = new FormData(); - fd.append('files', file); - xhr.send(fd); + // Шаг 1: распаковать ZIP через ВМ-буфер (паттерн drhider) + var token = crypto.randomUUID(); + var vmUrl = VM_UPLOAD_URL + token + '_0'; + var putResp = await fetch(vmUrl, { method: 'PUT', body: file, headers: { 'Content-Type': 'application/octet-stream' } }); + if (!putResp.ok) throw new Error('unzip failed: VM upload HTTP ' + putResp.status); + var refsResp = await fetch('/api/unzip_refs?_=' + Date.now(), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ files: [{ name: file.name, size: file.size, url: vmUrl }] }) }); + if (!refsResp.ok) throw new Error('unzip failed: HTTP ' + refsResp.status); + var zipResp = await refsResp.json(); if (!zipResp.ok || !zipResp.files) throw new Error('unzip failed'); // Подтверждение: показать первые 10 файлов + итог diff --git a/site/templates/index.html b/site/templates/index.html index 5daa513..1087b96 100644 --- a/site/templates/index.html +++ b/site/templates/index.html @@ -73,7 +73,7 @@
Nubes - Сверка договоров — LLM AI-driven Event Sourcing v2.0.10 + Сверка договоров — LLM AI-driven Event Sourcing v2.0.11
○ Загрузка→ ○ Классификация→ @@ -218,7 +218,7 @@ - +