From af172115b093a4400555bbcfc36fb6274b29e1d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Tue, 25 Aug 2026 08:37:09 +0300 Subject: [PATCH] =?UTF-8?q?fix(upload):=20=D0=BF=D0=B0=D1=80=D0=B0=D0=BC?= =?UTF-8?q?=D0=B5=D1=82=D1=80=D0=B8=D0=B7=D0=B0=D1=86=D0=B8=D1=8F=20pullRe?= =?UTF-8?q?tries/pullRetryDelay/pullTimeout,=20configure=20=D0=BB=D0=B8?= =?UTF-8?q?=D0=BC=D0=B8=D1=82=D0=BE=D0=B2/TTL,=20onStatus=20=D0=B2=20init,?= =?UTF-8?q?=20=D0=BB=D0=BE=D0=B3=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8?= =?UTF-8?q?=D0=B5=20sid/name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_upload_module.py | 11 +++----- upload/README.md | 4 +++ upload/backend/session/__init__.py | 3 ++- upload/backend/session/add_file.py | 8 +++--- upload/backend/session/state.py | 15 +++++++++++ upload/backend/upload_refs/blueprint.py | 29 ++++++++++++++++------ upload/backend/upload_refs/pull_file.py | 14 +++++++---- upload/config.example.json | 2 ++ upload/frontend/table/init_upload_table.js | 7 +++--- 9 files changed, 64 insertions(+), 29 deletions(-) diff --git a/tests/test_upload_module.py b/tests/test_upload_module.py index aa5f82a..e35b87d 100644 --- a/tests/test_upload_module.py +++ b/tests/test_upload_module.py @@ -44,12 +44,9 @@ def test_add_file_missing_session(): assert get_files("nonexistent") is None -def test_session_limit(monkeypatch): - # importlib: `import ...session.add_file as m` затеняется одноимённой функцией - import importlib - add_file_mod = importlib.import_module("upload.backend.session.add_file") - old = add_file_mod.MAX_SESSION_BYTES - add_file_mod.MAX_SESSION_BYTES = 10 +def test_session_limit(): + from upload.backend.session import configure + configure(max_session_bytes=10) try: sid = create_session() assert add_file(sid, "a", b"12345") is True @@ -57,7 +54,7 @@ def test_session_limit(monkeypatch): assert add_file(sid, "b", b"123456") is False assert file_count(sid) == 1 finally: - add_file_mod.MAX_SESSION_BYTES = old + configure(max_session_bytes=500 * 1024 * 1024) def test_store_result_and_csv(): diff --git a/upload/README.md b/upload/README.md index 1fc450f..eea2b74 100644 --- a/upload/README.md +++ b/upload/README.md @@ -86,6 +86,8 @@ app.register_blueprint(create_upload_refs_blueprint({ "apiPrefix": "/api", # префикс эндпоинтов "vmUploadPrefix": "https://.../drhider-upload/", # доверенный префикс (SSRF) "maxFileBytes": 50 * 1024 * 1024, + "maxSessionBytes": 500 * 1024 * 1024, + "ttlSeconds": 1800, "pullRetries": 3, "pullRetryDelay": 2, })) @@ -113,6 +115,8 @@ app.register_blueprint(create_upload_refs_blueprint({ | `maxFileBytes` / `maxSessionBytes` | лимиты 50 МБ / 500 МБ | | `apiPrefix` | префикс Blueprint `/api` | | `pullRetries` / `pullRetryDelay` | ретраи pull (3 × 2с) | +| `pullTimeout` | таймаут одного GET pull (сек) | +| `ttlSeconds` | TTL сессии (по умолчанию 1800) | | `estMbSec` | оценка времени обработки, сек/МБ (только UI) | --- diff --git a/upload/backend/session/__init__.py b/upload/backend/session/__init__.py index 6bb30ed..2fe81f2 100644 --- a/upload/backend/session/__init__.py +++ b/upload/backend/session/__init__.py @@ -14,7 +14,7 @@ from .store_csv import store_csv, get_csv from .ttl import touch, pause_ttl, resume_ttl from .cancel import request_cancel, get_cancel_event from .cleanup import cleanup -from .state import TTL_SECONDS, MAX_FILE_BYTES, MAX_SESSION_BYTES +from .state import TTL_SECONDS, MAX_FILE_BYTES, MAX_SESSION_BYTES, configure __all__ = [ "create_session", @@ -31,6 +31,7 @@ __all__ = [ "request_cancel", "get_cancel_event", "cleanup", + "configure", "TTL_SECONDS", "MAX_FILE_BYTES", "MAX_SESSION_BYTES", diff --git a/upload/backend/session/add_file.py b/upload/backend/session/add_file.py index 1aa1e0a..f598d63 100644 --- a/upload/backend/session/add_file.py +++ b/upload/backend/session/add_file.py @@ -1,6 +1,6 @@ """add_file — добавить файл в сессию (с проверкой суммарного лимита).""" -from .state import _sessions, _lock, MAX_SESSION_BYTES +from . import state def add_file(sid: str, filename: str, content: bytes) -> bool: @@ -14,12 +14,12 @@ def add_file(sid: str, filename: str, content: bytes) -> bool: Returns: True если добавлено; False если сессии нет или превышен лимит сессии. """ - with _lock: - s = _sessions.get(sid) + with state._lock: + s = state._sessions.get(sid) if not s: return False total = sum(len(c) for _, c in s["files"]) - if total + len(content) > MAX_SESSION_BYTES: + if total + len(content) > state.MAX_SESSION_BYTES: return False # превышен суммарный лимит сессии s["files"].append((filename, content)) return True diff --git a/upload/backend/session/state.py b/upload/backend/session/state.py index 633b3c7..dc70a72 100644 --- a/upload/backend/session/state.py +++ b/upload/backend/session/state.py @@ -28,3 +28,18 @@ def _start_timer(sid: str) -> threading.Timer: timer.daemon = True timer.start() return timer + + +def configure(max_file_bytes: int = None, max_session_bytes: int = None, + ttl_seconds: int = None): + """Переопределить лимиты/TTL из конфига приложения (глобально). + + None — оставить текущее значение. + """ + global MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS + if max_file_bytes is not None: + MAX_FILE_BYTES = max_file_bytes + if max_session_bytes is not None: + MAX_SESSION_BYTES = max_session_bytes + if ttl_seconds is not None: + TTL_SECONDS = ttl_seconds diff --git a/upload/backend/upload_refs/blueprint.py b/upload/backend/upload_refs/blueprint.py index efb4a03..6c6be94 100644 --- a/upload/backend/upload_refs/blueprint.py +++ b/upload/backend/upload_refs/blueprint.py @@ -14,7 +14,7 @@ import logging from flask import Blueprint, request, jsonify from ..session import (create_session, add_file, get_files, file_count, - MAX_FILE_BYTES) + MAX_FILE_BYTES, configure) from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX from .safe_name import safe_name from .pull_file import pull_file @@ -26,15 +26,28 @@ def create_upload_refs_blueprint(cfg: dict) -> Blueprint: """Создать Blueprint с эндпоинтом upload_refs. cfg (все ключи опциональны, есть дефолты): - apiPrefix (str) — префикс Blueprint, по умолчанию "/api" - vmUploadPrefix (str) — доверенный префикс ВМ-буфера (SSRF-валидация) - maxFileBytes (int) — лимит на один файл - pullRetries (int) — ретраи pull - pullRetryDelay (int) — пауза между ретраями + apiPrefix (str) — префикс Blueprint, по умолчанию "/api" + vmUploadPrefix (str) — доверенный префикс ВМ-буфера (SSRF-валидация) + maxFileBytes (int) — лимит на один файл + maxSessionBytes (int) — суммарный лимит сессии (применяется к сессиям) + ttlSeconds (int) — TTL сессии + pullRetries (int) — ретраи pull + pullRetryDelay (int) — пауза между ретраями (сек) + pullTimeout (int) — таймаут одного GET pull """ prefix = cfg.get("apiPrefix", "/api") vm_prefix = cfg.get("vmUploadPrefix", VM_UPLOAD_PREFIX) max_file_bytes = cfg.get("maxFileBytes", MAX_FILE_BYTES) + pull_retries = cfg.get("pullRetries", PULL_RETRIES) + pull_delay = cfg.get("pullRetryDelay", PULL_RETRY_DELAY) + pull_timeout = cfg.get("pullTimeout", 120) + + # Применить лимиты сессии/TTL из конфига (глобально для всех сессий) + configure( + max_file_bytes=cfg.get("maxFileBytes"), + max_session_bytes=cfg.get("maxSessionBytes"), + ttl_seconds=cfg.get("ttlSeconds"), + ) bp = Blueprint("upload_refs", __name__, url_prefix=prefix) @@ -55,7 +68,7 @@ def create_upload_refs_blueprint(cfg: dict) -> Blueprint: return jsonify({"ok": False, "error": "No files"}), 400 added = 0 try: - with httpx.Client(timeout=120, follow_redirects=True) as client: + with httpx.Client(timeout=pull_timeout, follow_redirects=True) as client: for ref in refs: name = safe_name(ref.get("name") or "") url = ref.get("url") @@ -75,7 +88,7 @@ def create_upload_refs_blueprint(cfg: dict) -> Blueprint: pass continue # Pull с ретраями: разовые DNS/сетевые сбои не роняют всю загрузку - content = pull_file(client, url) # бросает при неудаче всех попыток + content = pull_file(client, url, pull_retries, pull_delay, sid=sid, name=name) # бросает при неудаче всех попыток log.info("upload_refs: pulled sid=%s file=%r size=%d", sid, name, len(content)) if len(content) > max_file_bytes: log.warning("upload_refs: pulled file exceeds %dMB, skip sid=%s file=%r size=%d", diff --git a/upload/backend/upload_refs/pull_file.py b/upload/backend/upload_refs/pull_file.py index 583ce02..9fab75c 100644 --- a/upload/backend/upload_refs/pull_file.py +++ b/upload/backend/upload_refs/pull_file.py @@ -8,12 +8,16 @@ from .config import PULL_RETRIES, PULL_RETRY_DELAY log = logging.getLogger("upload.upload_refs.pull") -def pull_file(client, url: str) -> bytes: +def pull_file(client, url: str, retries: int = PULL_RETRIES, + delay: float = PULL_RETRY_DELAY, sid: str = None, name: str = None) -> bytes: """GET url с ретраями; читает по частям (stream). Args: client: httpx.Client url: URL файла на ВМ-буфере. + retries: число попыток. + delay: пауза между попытками (сек). + sid/name: для логирования (опционально). Returns: Содержимое файла (bytes). @@ -22,14 +26,14 @@ def pull_file(client, url: str) -> bytes: Последнюю ошибку попытки, если все ретраи не удались. """ last_err = None - for attempt in range(PULL_RETRIES): + for attempt in range(retries): try: with client.stream("GET", url) as resp: resp.raise_for_status() return b"".join(resp.iter_bytes()) except Exception as e: last_err = e - log.warning("pull: attempt %d/%d failed url=%r: %r", - attempt + 1, PULL_RETRIES, url, e) - time.sleep(PULL_RETRY_DELAY) + log.warning("pull: attempt %d/%d failed sid=%s file=%r: %r", + attempt + 1, retries, sid, name, e) + time.sleep(delay) raise last_err if last_err else RuntimeError("pull failed") diff --git a/upload/config.example.json b/upload/config.example.json index fa74e39..037494d 100644 --- a/upload/config.example.json +++ b/upload/config.example.json @@ -7,5 +7,7 @@ "vmUploadPrefix": "https://contracts.kube5s.ru/drhider-upload/", "pullRetries": 3, "pullRetryDelay": 2, + "pullTimeout": 120, + "ttlSeconds": 1800, "estMbSec": 12 } diff --git a/upload/frontend/table/init_upload_table.js b/upload/frontend/table/init_upload_table.js index 2cc560e..1d9aacd 100644 --- a/upload/frontend/table/init_upload_table.js +++ b/upload/frontend/table/init_upload_table.js @@ -1,8 +1,7 @@ // initUploadTable — сборка слоя 1 (выбор файлов/папки/архива). // Состояние (files/fileMeta/overNames) живёт внутри модуля. -import { addFiles as addFilesToState } from './on_files_change.js'; -import { makeFilesChangeHandler } from './on_files_change.js'; +import { addFiles as addFilesToState, makeFilesChangeHandler } from './on_files_change.js'; import { makeFolderChangeHandler } from './on_folder_change.js'; import { render } from './render.js'; import { setStatus } from './set_status.js'; @@ -20,8 +19,8 @@ export function initUploadTable(cfg, els) { proc: null, // {phase, procState, procExtractRate} — задаёт слой 3 }; - const onFilesChange = makeFilesChangeHandler({ state, cfg, els }); - const onFolderChange = makeFolderChangeHandler({ state, cfg, els }); + const onFilesChange = makeFilesChangeHandler({ state, cfg, els, onStatus: els.onStatus }); + const onFolderChange = makeFolderChangeHandler({ state, cfg, els, onStatus: els.onStatus }); els.fileInput.addEventListener('change', onFilesChange); els.folderInput.addEventListener('change', onFolderChange);