Nail/drhider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(upload): параметризация pullRetries/pullRetryDelay/pullTimeout, configure лимитов/TTL, onStatus в init, логирование sid/name

Browse Source
This commit is contained in:
“Naeel”
2026-08-25 08:37:09 +03:00
parent 5711e97fe5
commit af172115b0
9 changed files with 64 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_upload_module.py
+4 -7
View File
@@ -44,12 +44,9 @@ def test_add_file_missing_session():
assert get_files("nonexistent") is None
def test_session_limit(monkeypatch):
# importlib: `import ...session.add_file as m` затеняется одноимённой функцией
import importlib
add_file_mod = importlib.import_module("upload.backend.session.add_file")
old = add_file_mod.MAX_SESSION_BYTES
add_file_mod.MAX_SESSION_BYTES = 10
def test_session_limit():
from upload.backend.session import configure
configure(max_session_bytes=10)
try:
sid = create_session()
assert add_file(sid, "a", b"12345") is True
@@ -57,7 +54,7 @@ def test_session_limit(monkeypatch):
assert add_file(sid, "b", b"123456") is False
assert file_count(sid) == 1
finally:
add_file_mod.MAX_SESSION_BYTES = old
configure(max_session_bytes=500 * 1024 * 1024)
def test_store_result_and_csv():
upload/README.md
+4
View File
@@ -86,6 +86,8 @@ app.register_blueprint(create_upload_refs_blueprint({
"apiPrefix": "/api", # префикс эндпоинтов
"vmUploadPrefix": "https://.../drhider-upload/", # доверенный префикс (SSRF)
"maxFileBytes": 50 * 1024 * 1024,
"maxSessionBytes": 500 * 1024 * 1024,
"ttlSeconds": 1800,
"pullRetries": 3,
"pullRetryDelay": 2,
}))
@@ -113,6 +115,8 @@ app.register_blueprint(create_upload_refs_blueprint({
| `maxFileBytes` / `maxSessionBytes` | лимиты 50 МБ / 500 МБ |
| `apiPrefix` | префикс Blueprint `/api` |
| `pullRetries` / `pullRetryDelay` | ретраи pull (3 × 2с) |
| `pullTimeout` | таймаут одного GET pull (сек) |
| `ttlSeconds` | TTL сессии (по умолчанию 1800) |
| `estMbSec` | оценка времени обработки, сек/МБ (только UI) |
---
upload/backend/session/__init__.py
+2 -1
View File
@@ -14,7 +14,7 @@ from .store_csv import store_csv, get_csv
from .ttl import touch, pause_ttl, resume_ttl
from .cancel import request_cancel, get_cancel_event
from .cleanup import cleanup
from .state import TTL_SECONDS, MAX_FILE_BYTES, MAX_SESSION_BYTES
from .state import TTL_SECONDS, MAX_FILE_BYTES, MAX_SESSION_BYTES, configure
__all__ = [
"create_session",
@@ -31,6 +31,7 @@ __all__ = [
"request_cancel",
"get_cancel_event",
"cleanup",
"configure",
"TTL_SECONDS",
"MAX_FILE_BYTES",
"MAX_SESSION_BYTES",
upload/backend/session/add_file.py
+4 -4
View File
@@ -1,6 +1,6 @@
"""add_file — добавить файл в сессию (с проверкой суммарного лимита)."""
from .state import _sessions, _lock, MAX_SESSION_BYTES
from . import state
def add_file(sid: str, filename: str, content: bytes) -> bool:
@@ -14,12 +14,12 @@ def add_file(sid: str, filename: str, content: bytes) -> bool:
Returns:
True если добавлено; False если сессии нет или превышен лимит сессии.
"""
with _lock:
s = _sessions.get(sid)
with state._lock:
s = state._sessions.get(sid)
if not s:
return False
total = sum(len(c) for _, c in s["files"])
if total + len(content) > MAX_SESSION_BYTES:
if total + len(content) > state.MAX_SESSION_BYTES:
return False # превышен суммарный лимит сессии
s["files"].append((filename, content))
return True
upload/backend/session/state.py
+15
View File
@@ -28,3 +28,18 @@ def _start_timer(sid: str) -> threading.Timer:
timer.daemon = True
timer.start()
return timer
def configure(max_file_bytes: int = None, max_session_bytes: int = None,
ttl_seconds: int = None):
"""Переопределить лимиты/TTL из конфига приложения (глобально).
None — оставить текущее значение.
"""
global MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS
if max_file_bytes is not None:
MAX_FILE_BYTES = max_file_bytes
if max_session_bytes is not None:
MAX_SESSION_BYTES = max_session_bytes
if ttl_seconds is not None:
TTL_SECONDS = ttl_seconds
upload/backend/upload_refs/blueprint.py
+21 -8
View File
@@ -14,7 +14,7 @@ import logging
from flask import Blueprint, request, jsonify
from ..session import (create_session, add_file, get_files, file_count,
MAX_FILE_BYTES)
MAX_FILE_BYTES, configure)
from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX
from .safe_name import safe_name
from .pull_file import pull_file
@@ -26,15 +26,28 @@ def create_upload_refs_blueprint(cfg: dict) -> Blueprint:
"""Создать Blueprint с эндпоинтом upload_refs.
cfg (все ключи опциональны, есть дефолты):
apiPrefix (str) — префикс Blueprint, по умолчанию "/api"
vmUploadPrefix (str) — доверенный префикс ВМ-буфера (SSRF-валидация)
maxFileBytes (int) — лимит на один файл
pullRetries (int) — ретраи pull
pullRetryDelay (int) — пауза между ретраями
apiPrefix (str) — префикс Blueprint, по умолчанию "/api"
vmUploadPrefix (str) — доверенный префикс ВМ-буфера (SSRF-валидация)
maxFileBytes (int) — лимит на один файл
maxSessionBytes (int) — суммарный лимит сессии (применяется к сессиям)
ttlSeconds (int) — TTL сессии
pullRetries (int) — ретраи pull
pullRetryDelay (int) — пауза между ретраями (сек)
pullTimeout (int) — таймаут одного GET pull
"""
prefix = cfg.get("apiPrefix", "/api")
vm_prefix = cfg.get("vmUploadPrefix", VM_UPLOAD_PREFIX)
max_file_bytes = cfg.get("maxFileBytes", MAX_FILE_BYTES)
pull_retries = cfg.get("pullRetries", PULL_RETRIES)
pull_delay = cfg.get("pullRetryDelay", PULL_RETRY_DELAY)
pull_timeout = cfg.get("pullTimeout", 120)
# Применить лимиты сессии/TTL из конфига (глобально для всех сессий)
configure(
max_file_bytes=cfg.get("maxFileBytes"),
max_session_bytes=cfg.get("maxSessionBytes"),
ttl_seconds=cfg.get("ttlSeconds"),
)
bp = Blueprint("upload_refs", __name__, url_prefix=prefix)
@@ -55,7 +68,7 @@ def create_upload_refs_blueprint(cfg: dict) -> Blueprint:
return jsonify({"ok": False, "error": "No files"}), 400
added = 0
try:
with httpx.Client(timeout=120, follow_redirects=True) as client:
with httpx.Client(timeout=pull_timeout, follow_redirects=True) as client:
for ref in refs:
name = safe_name(ref.get("name") or "")
url = ref.get("url")
@@ -75,7 +88,7 @@ def create_upload_refs_blueprint(cfg: dict) -> Blueprint:
pass
continue
# Pull с ретраями: разовые DNS/сетевые сбои не роняют всю загрузку
content = pull_file(client, url) # бросает при неудаче всех попыток
content = pull_file(client, url, pull_retries, pull_delay, sid=sid, name=name) # бросает при неудаче всех попыток
log.info("upload_refs: pulled sid=%s file=%r size=%d", sid, name, len(content))
if len(content) > max_file_bytes:
log.warning("upload_refs: pulled file exceeds %dMB, skip sid=%s file=%r size=%d",
upload/backend/upload_refs/pull_file.py
+9 -5
View File
@@ -8,12 +8,16 @@ from .config import PULL_RETRIES, PULL_RETRY_DELAY
log = logging.getLogger("upload.upload_refs.pull")
def pull_file(client, url: str) -> bytes:
def pull_file(client, url: str, retries: int = PULL_RETRIES,
delay: float = PULL_RETRY_DELAY, sid: str = None, name: str = None) -> bytes:
"""GET url с ретраями; читает по частям (stream).
Args:
client: httpx.Client
url: URL файла на ВМ-буфере.
retries: число попыток.
delay: пауза между попытками (сек).
sid/name: для логирования (опционально).
Returns:
Содержимое файла (bytes).
@@ -22,14 +26,14 @@ def pull_file(client, url: str) -> bytes:
Последнюю ошибку попытки, если все ретраи не удались.
"""
last_err = None
for attempt in range(PULL_RETRIES):
for attempt in range(retries):
try:
with client.stream("GET", url) as resp:
resp.raise_for_status()
return b"".join(resp.iter_bytes())
except Exception as e:
last_err = e
log.warning("pull: attempt %d/%d failed url=%r: %r",
attempt + 1, PULL_RETRIES, url, e)
time.sleep(PULL_RETRY_DELAY)
log.warning("pull: attempt %d/%d failed sid=%s file=%r: %r",
attempt + 1, retries, sid, name, e)
time.sleep(delay)
raise last_err if last_err else RuntimeError("pull failed")
upload/config.example.json
+2
View File
@@ -7,5 +7,7 @@
"vmUploadPrefix": "https://contracts.kube5s.ru/drhider-upload/",
"pullRetries": 3,
"pullRetryDelay": 2,
"pullTimeout": 120,
"ttlSeconds": 1800,
"estMbSec": 12
}
upload/frontend/table/init_upload_table.js
+3 -4
View File
@@ -1,8 +1,7 @@
// initUploadTable — сборка слоя 1 (выбор файлов/папки/архива).
// Состояние (files/fileMeta/overNames) живёт внутри модуля.
import { addFiles as addFilesToState } from './on_files_change.js';
import { makeFilesChangeHandler } from './on_files_change.js';
import { addFiles as addFilesToState, makeFilesChangeHandler } from './on_files_change.js';
import { makeFolderChangeHandler } from './on_folder_change.js';
import { render } from './render.js';
import { setStatus } from './set_status.js';
@@ -20,8 +19,8 @@ export function initUploadTable(cfg, els) {
proc: null, // {phase, procState, procExtractRate} — задаёт слой 3
};
const onFilesChange = makeFilesChangeHandler({ state, cfg, els });
const onFolderChange = makeFolderChangeHandler({ state, cfg, els });
const onFilesChange = makeFilesChangeHandler({ state, cfg, els, onStatus: els.onStatus });
const onFolderChange = makeFolderChangeHandler({ state, cfg, els, onStatus: els.onStatus });
els.fileInput.addEventListener('change', onFilesChange);
els.folderInput.addEventListener('change', onFolderChange);