Implement reusable upload platform

upload/backend/__init__.py

@@ -0,0 +1 @@
+"""Backend-модули переиспользуемого слоя загрузки."""

upload/backend/session/__init__.py

@@ -0,0 +1,19 @@
+"""Самодостаточное in-memory хранилище сессий."""
+
+from .add_file import add_file
+from .cleanup import cleanup
+from .create_session import create_session
+from .get_files import file_count, get_files
+from .state import MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS, configure
+
+__all__ = [
+    "create_session",
+    "add_file",
+    "get_files",
+    "file_count",
+    "cleanup",
+    "configure",
+    "TTL_SECONDS",
+    "MAX_FILE_BYTES",
+    "MAX_SESSION_BYTES",
+]

upload/backend/session/add_file.py

@@ -0,0 +1,17 @@
+"""Добавление файла в сессию."""
+
+from . import state
+
+
+def add_file(sid: str, filename: str, content: bytes) -> bool:
+    """Добавить файл, если сессия существует и общий лимит не превышен."""
+
+    with state._lock:
+        session = state._sessions.get(sid)
+        if not session:
+            return False
+        total = sum(len(item_content) for _, item_content in session["files"])
+        if total + len(content) > state.MAX_SESSION_BYTES:
+            return False
+        session["files"].append((filename, content))
+        return True

upload/backend/session/cleanup.py

@@ -0,0 +1,12 @@
+"""Удаление сессии."""
+
+from .state import _lock, _sessions
+
+
+def cleanup(sid: str):
+    """Удалить сессию и остановить её TTL-таймер."""
+
+    with _lock:
+        session = _sessions.pop(sid, None)
+        if session and session.get("timer"):
+            session["timer"].cancel()

upload/backend/session/create_session.py

@@ -0,0 +1,19 @@
+"""Создание сессии."""
+
+import threading
+import uuid
+
+from .state import _lock, _sessions, _start_timer
+
+
+def create_session() -> str:
+    """Создать сессию и вернуть её уникальный идентификатор."""
+
+    sid = uuid.uuid4().hex
+    with _lock:
+        _sessions[sid] = {
+            "files": [],
+            "timer": _start_timer(sid),
+            "cancel": threading.Event(),
+        }
+    return sid

upload/backend/session/get_files.py

@@ -0,0 +1,21 @@
+"""Чтение файлов сессии."""
+
+from typing import List, Optional, Tuple
+
+from .state import _lock, _sessions
+
+
+def get_files(sid: str) -> Optional[List[Tuple[str, bytes]]]:
+    """Вернуть файлы сессии или None, если сессия не найдена."""
+
+    with _lock:
+        session = _sessions.get(sid)
+        return list(session["files"]) if session else None
+
+
+def file_count(sid: str) -> int:
+    """Вернуть количество файлов в сессии."""
+
+    with _lock:
+        session = _sessions.get(sid)
+        return len(session["files"]) if session else 0

upload/backend/session/state.py

@@ -0,0 +1,37 @@
+"""Общее состояние сессий, блокировка, лимиты и TTL."""
+
+import threading
+
+
+TTL_SECONDS = 30 * 60
+MAX_FILE_BYTES = 50 * 1024 * 1024
+MAX_SESSION_BYTES = 500 * 1024 * 1024
+
+_sessions: dict = {}
+_lock = threading.Lock()
+
+
+def _start_timer(sid: str) -> threading.Timer:
+    """Запустить таймер автоочистки сессии через TTL."""
+
+    def _clean():
+        with _lock:
+            _sessions.pop(sid, None)
+
+    timer = threading.Timer(TTL_SECONDS, _clean)
+    timer.daemon = True
+    timer.start()
+    return timer
+
+
+def configure(max_file_bytes: int = None, max_session_bytes: int = None,
+              ttl_seconds: int = None):
+    """Переопределить лимиты и TTL из конфигурации приложения."""
+
+    global MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS
+    if max_file_bytes is not None:
+        MAX_FILE_BYTES = max_file_bytes
+    if max_session_bytes is not None:
+        MAX_SESSION_BYTES = max_session_bytes
+    if ttl_seconds is not None:
+        TTL_SECONDS = ttl_seconds

upload/backend/upload_refs/__init__.py

@@ -0,0 +1,15 @@
+"""Переиспользуемый backend-слой закачки через ВМ."""
+
+from .blueprint import create_upload_refs_blueprint
+from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX
+from .pull_file import pull_file
+from .safe_name import safe_name
+
+__all__ = [
+    "create_upload_refs_blueprint",
+    "safe_name",
+    "pull_file",
+    "PULL_RETRIES",
+    "PULL_RETRY_DELAY",
+    "VM_UPLOAD_PREFIX",
+]

upload/backend/upload_refs/blueprint.py

@@ -0,0 +1,79 @@
+"""Blueprint слоя закачки ссылок через ВМ-буфер."""
+
+import logging
+
+import requests
+from flask import Blueprint, jsonify, request
+
+from ..session import (MAX_FILE_BYTES, add_file, configure, create_session,
+                       file_count, get_files)
+from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX
+from .pull_file import pull_file
+from .safe_name import safe_name
+
+
+log = logging.getLogger("upload.upload_refs")
+
+
+def create_upload_refs_blueprint(cfg: dict) -> Blueprint:
+    """Создать Blueprint с POST ``/upload_refs``."""
+
+    cfg = cfg or {}
+    prefix = cfg.get("apiPrefix", "/api")
+    vm_prefix = cfg.get("vmUploadUrl", VM_UPLOAD_PREFIX)
+    max_file_bytes = cfg.get("maxFileBytes", MAX_FILE_BYTES)
+    pull_retries = cfg.get("pullRetries", PULL_RETRIES)
+    pull_delay = cfg.get("pullRetryDelay", PULL_RETRY_DELAY)
+    pull_timeout = cfg.get("pullTimeout", 120)
+    configure(max_session_bytes=cfg.get("maxSessionBytes"))
+
+    blueprint = Blueprint("upload_refs", __name__, url_prefix=prefix)
+
+    @blueprint.route("/upload_refs", methods=["POST"])
+    def upload_refs():
+        data = request.get_json(silent=True) or {}
+        sid = data.get("session") or create_session()
+        refs = data.get("files") or []
+        if not isinstance(refs, list) or not refs:
+            return jsonify({"ok": False, "error": "No files"}), 400
+
+        added = 0
+        try:
+            with requests.Session() as client:
+                for ref in refs:
+                    if not isinstance(ref, dict):
+                        continue
+                    name = safe_name(ref.get("name", ""))
+                    url = ref.get("url")
+                    if not name or not isinstance(url, str) or not url.startswith(vm_prefix):
+                        continue
+                    if (ref.get("size") or 0) > max_file_bytes:
+                        _delete(client, url)
+                        continue
+                    content = pull_file(
+                        client, url, max_file_bytes, pull_retries, pull_delay,
+                        pull_timeout, sid=sid, name=name,
+                    )
+                    if not add_file(sid, name, content):
+                        if get_files(sid) is None:
+                            return jsonify({"ok": False, "error": "Session not found"}), 404
+                        _delete(client, url)
+                        continue
+                    _delete(client, url)
+                    added += 1
+        except Exception as error:
+            log.error("upload_refs: pull error sid=%s: %r", sid, error)
+            return jsonify({"ok": False, "error": "Pull failed: %s" % error}), 502
+
+        return jsonify({"ok": True, "session": sid, "count": file_count(sid)})
+
+    return blueprint
+
+
+def _delete(client, url: str):
+    """Удалить временный объект на ВМ, не ломая основной запрос при сбое."""
+
+    try:
+        client.delete(url, timeout=10)
+    except Exception:
+        log.warning("upload_refs: could not delete VM object url=%r", url)

upload/backend/upload_refs/config.py

@@ -0,0 +1,5 @@
+"""Значения по умолчанию для pull из ВМ-буфера."""
+
+PULL_RETRIES = 3
+PULL_RETRY_DELAY = 2
+VM_UPLOAD_PREFIX = "https://example.invalid/upload/"

upload/backend/upload_refs/pull_file.py

@@ -0,0 +1,38 @@
+"""Исходящий pull файла из ВМ-буфера с ретраями и лимитом размера."""
+
+import logging
+import time
+
+from .config import PULL_RETRIES, PULL_RETRY_DELAY
+
+
+log = logging.getLogger("upload.upload_refs.pull")
+
+
+def pull_file(client, url: str, max_bytes: int,
+              retries: int = PULL_RETRIES, delay: float = PULL_RETRY_DELAY,
+              timeout: int = 120, sid: str = None, name: str = None) -> bytes:
+    """Забрать файл потоково, не принимая тело больше ``max_bytes``."""
+
+    last_error = None
+    for attempt in range(retries):
+        try:
+            response = client.get(url, stream=True, timeout=timeout)
+            response.raise_for_status()
+            chunks = []
+            total = 0
+            for chunk in response.iter_content(chunk_size=1024 * 1024):
+                if not chunk:
+                    continue
+                total += len(chunk)
+                if total > max_bytes:
+                    raise ValueError("file exceeds maxFileBytes")
+                chunks.append(chunk)
+            return b"".join(chunks)
+        except Exception as error:
+            last_error = error
+            log.warning("pull: attempt %d/%d failed sid=%s file=%r: %r",
+                        attempt + 1, retries, sid, name, error)
+            if attempt + 1 < retries:
+                time.sleep(delay)
+    raise last_error if last_error else RuntimeError("pull failed")

upload/backend/upload_refs/safe_name.py

@@ -0,0 +1,13 @@
+"""Санитизация имени файла с сохранением подпапок."""
+
+
+def safe_name(name: str) -> str:
+    """Вернуть безопасное относительное имя или пустую строку."""
+
+    if not isinstance(name, str) or not name:
+        return ""
+    parts = [part for part in name.replace("\\", "/").split("/")
+             if part and part != "."]
+    if not parts or any(part == ".." for part in parts):
+        return ""
+    return "/".join(parts)