Remove VM upload layer; keep file picker only
This commit is contained in:
“Naeel”
@@ -1 +0,0 @@
|
||||
"""Backend-модули переиспользуемого слоя загрузки."""
|
||||
@@ -1,19 +0,0 @@
|
||||
"""Самодостаточное in-memory хранилище сессий."""
|
||||
|
||||
from .add_file import add_file
|
||||
from .cleanup import cleanup
|
||||
from .create_session import create_session
|
||||
from .get_files import file_count, get_files
|
||||
from .state import MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS, configure
|
||||
|
||||
__all__ = [
|
||||
"create_session",
|
||||
"add_file",
|
||||
"get_files",
|
||||
"file_count",
|
||||
"cleanup",
|
||||
"configure",
|
||||
"TTL_SECONDS",
|
||||
"MAX_FILE_BYTES",
|
||||
"MAX_SESSION_BYTES",
|
||||
]
|
||||
@@ -1,17 +0,0 @@
|
||||
"""Добавление файла в сессию."""
|
||||
|
||||
from . import state
|
||||
|
||||
|
||||
def add_file(sid: str, filename: str, content: bytes) -> bool:
|
||||
"""Добавить файл, если сессия существует и общий лимит не превышен."""
|
||||
|
||||
with state._lock:
|
||||
session = state._sessions.get(sid)
|
||||
if not session:
|
||||
return False
|
||||
total = sum(len(item_content) for _, item_content in session["files"])
|
||||
if total + len(content) > state.MAX_SESSION_BYTES:
|
||||
return False
|
||||
session["files"].append((filename, content))
|
||||
return True
|
||||
@@ -1,12 +0,0 @@
|
||||
"""Удаление сессии."""
|
||||
|
||||
from .state import _lock, _sessions
|
||||
|
||||
|
||||
def cleanup(sid: str):
|
||||
"""Удалить сессию и остановить её TTL-таймер."""
|
||||
|
||||
with _lock:
|
||||
session = _sessions.pop(sid, None)
|
||||
if session and session.get("timer"):
|
||||
session["timer"].cancel()
|
||||
@@ -1,19 +0,0 @@
|
||||
"""Создание сессии."""
|
||||
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
from .state import _lock, _sessions, _start_timer
|
||||
|
||||
|
||||
def create_session() -> str:
|
||||
"""Создать сессию и вернуть её уникальный идентификатор."""
|
||||
|
||||
sid = uuid.uuid4().hex
|
||||
with _lock:
|
||||
_sessions[sid] = {
|
||||
"files": [],
|
||||
"timer": _start_timer(sid),
|
||||
"cancel": threading.Event(),
|
||||
}
|
||||
return sid
|
||||
@@ -1,21 +0,0 @@
|
||||
"""Чтение файлов сессии."""
|
||||
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from .state import _lock, _sessions
|
||||
|
||||
|
||||
def get_files(sid: str) -> Optional[List[Tuple[str, bytes]]]:
|
||||
"""Вернуть файлы сессии или None, если сессия не найдена."""
|
||||
|
||||
with _lock:
|
||||
session = _sessions.get(sid)
|
||||
return list(session["files"]) if session else None
|
||||
|
||||
|
||||
def file_count(sid: str) -> int:
|
||||
"""Вернуть количество файлов в сессии."""
|
||||
|
||||
with _lock:
|
||||
session = _sessions.get(sid)
|
||||
return len(session["files"]) if session else 0
|
||||
@@ -1,37 +0,0 @@
|
||||
"""Общее состояние сессий, блокировка, лимиты и TTL."""
|
||||
|
||||
import threading
|
||||
|
||||
|
||||
TTL_SECONDS = 30 * 60
|
||||
MAX_FILE_BYTES = 50 * 1024 * 1024
|
||||
MAX_SESSION_BYTES = 500 * 1024 * 1024
|
||||
|
||||
_sessions: dict = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def _start_timer(sid: str) -> threading.Timer:
|
||||
"""Запустить таймер автоочистки сессии через TTL."""
|
||||
|
||||
def _clean():
|
||||
with _lock:
|
||||
_sessions.pop(sid, None)
|
||||
|
||||
timer = threading.Timer(TTL_SECONDS, _clean)
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
return timer
|
||||
|
||||
|
||||
def configure(max_file_bytes: int = None, max_session_bytes: int = None,
|
||||
ttl_seconds: int = None):
|
||||
"""Переопределить лимиты и TTL из конфигурации приложения."""
|
||||
|
||||
global MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS
|
||||
if max_file_bytes is not None:
|
||||
MAX_FILE_BYTES = max_file_bytes
|
||||
if max_session_bytes is not None:
|
||||
MAX_SESSION_BYTES = max_session_bytes
|
||||
if ttl_seconds is not None:
|
||||
TTL_SECONDS = ttl_seconds
|
||||
@@ -1,15 +0,0 @@
|
||||
"""Переиспользуемый backend-слой закачки через ВМ."""
|
||||
|
||||
from .blueprint import create_upload_refs_blueprint
|
||||
from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX
|
||||
from .pull_file import pull_file
|
||||
from .safe_name import safe_name
|
||||
|
||||
__all__ = [
|
||||
"create_upload_refs_blueprint",
|
||||
"safe_name",
|
||||
"pull_file",
|
||||
"PULL_RETRIES",
|
||||
"PULL_RETRY_DELAY",
|
||||
"VM_UPLOAD_PREFIX",
|
||||
]
|
||||
@@ -1,79 +0,0 @@
|
||||
"""Blueprint слоя закачки ссылок через ВМ-буфер."""
|
||||
|
||||
import logging
|
||||
|
||||
import requests
|
||||
from flask import Blueprint, jsonify, request
|
||||
|
||||
from ..session import (MAX_FILE_BYTES, add_file, configure, create_session,
|
||||
file_count, get_files)
|
||||
from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX
|
||||
from .pull_file import pull_file
|
||||
from .safe_name import safe_name
|
||||
|
||||
|
||||
log = logging.getLogger("upload.upload_refs")
|
||||
|
||||
|
||||
def create_upload_refs_blueprint(cfg: dict) -> Blueprint:
|
||||
"""Создать Blueprint с POST ``/upload_refs``."""
|
||||
|
||||
cfg = cfg or {}
|
||||
prefix = cfg.get("apiPrefix", "/api")
|
||||
vm_prefix = cfg.get("vmUploadUrl", VM_UPLOAD_PREFIX)
|
||||
max_file_bytes = cfg.get("maxFileBytes", MAX_FILE_BYTES)
|
||||
pull_retries = cfg.get("pullRetries", PULL_RETRIES)
|
||||
pull_delay = cfg.get("pullRetryDelay", PULL_RETRY_DELAY)
|
||||
pull_timeout = cfg.get("pullTimeout", 120)
|
||||
configure(max_session_bytes=cfg.get("maxSessionBytes"))
|
||||
|
||||
blueprint = Blueprint("upload_refs", __name__, url_prefix=prefix)
|
||||
|
||||
@blueprint.route("/upload_refs", methods=["POST"])
|
||||
def upload_refs():
|
||||
data = request.get_json(silent=True) or {}
|
||||
sid = data.get("session") or create_session()
|
||||
refs = data.get("files") or []
|
||||
if not isinstance(refs, list) or not refs:
|
||||
return jsonify({"ok": False, "error": "No files"}), 400
|
||||
|
||||
added = 0
|
||||
try:
|
||||
with requests.Session() as client:
|
||||
for ref in refs:
|
||||
if not isinstance(ref, dict):
|
||||
continue
|
||||
name = safe_name(ref.get("name", ""))
|
||||
url = ref.get("url")
|
||||
if not name or not isinstance(url, str) or not url.startswith(vm_prefix):
|
||||
continue
|
||||
if (ref.get("size") or 0) > max_file_bytes:
|
||||
_delete(client, url)
|
||||
continue
|
||||
content = pull_file(
|
||||
client, url, max_file_bytes, pull_retries, pull_delay,
|
||||
pull_timeout, sid=sid, name=name,
|
||||
)
|
||||
if not add_file(sid, name, content):
|
||||
if get_files(sid) is None:
|
||||
return jsonify({"ok": False, "error": "Session not found"}), 404
|
||||
_delete(client, url)
|
||||
continue
|
||||
_delete(client, url)
|
||||
added += 1
|
||||
except Exception as error:
|
||||
log.error("upload_refs: pull error sid=%s: %r", sid, error)
|
||||
return jsonify({"ok": False, "error": "Pull failed: %s" % error}), 502
|
||||
|
||||
return jsonify({"ok": True, "session": sid, "count": file_count(sid)})
|
||||
|
||||
return blueprint
|
||||
|
||||
|
||||
def _delete(client, url: str):
|
||||
"""Удалить временный объект на ВМ, не ломая основной запрос при сбое."""
|
||||
|
||||
try:
|
||||
client.delete(url, timeout=10)
|
||||
except Exception:
|
||||
log.warning("upload_refs: could not delete VM object url=%r", url)
|
||||
@@ -1,5 +0,0 @@
|
||||
"""Значения по умолчанию для pull из ВМ-буфера."""
|
||||
|
||||
PULL_RETRIES = 3
|
||||
PULL_RETRY_DELAY = 2
|
||||
VM_UPLOAD_PREFIX = "https://example.invalid/upload/"
|
||||
@@ -1,38 +0,0 @@
|
||||
"""Исходящий pull файла из ВМ-буфера с ретраями и лимитом размера."""
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from .config import PULL_RETRIES, PULL_RETRY_DELAY
|
||||
|
||||
|
||||
log = logging.getLogger("upload.upload_refs.pull")
|
||||
|
||||
|
||||
def pull_file(client, url: str, max_bytes: int,
|
||||
retries: int = PULL_RETRIES, delay: float = PULL_RETRY_DELAY,
|
||||
timeout: int = 120, sid: str = None, name: str = None) -> bytes:
|
||||
"""Забрать файл потоково, не принимая тело больше ``max_bytes``."""
|
||||
|
||||
last_error = None
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
response = client.get(url, stream=True, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
chunks = []
|
||||
total = 0
|
||||
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
raise ValueError("file exceeds maxFileBytes")
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
except Exception as error:
|
||||
last_error = error
|
||||
log.warning("pull: attempt %d/%d failed sid=%s file=%r: %r",
|
||||
attempt + 1, retries, sid, name, error)
|
||||
if attempt + 1 < retries:
|
||||
time.sleep(delay)
|
||||
raise last_error if last_error else RuntimeError("pull failed")
|
||||
@@ -1,13 +0,0 @@
|
||||
"""Санитизация имени файла с сохранением подпапок."""
|
||||
|
||||
|
||||
def safe_name(name: str) -> str:
|
||||
"""Вернуть безопасное относительное имя или пустую строку."""
|
||||
|
||||
if not isinstance(name, str) or not name:
|
||||
return ""
|
||||
parts = [part for part in name.replace("\\", "/").split("/")
|
||||
if part and part != "."]
|
||||
if not parts or any(part == ".." for part in parts):
|
||||
return ""
|
||||
return "/".join(parts)
|
||||
Reference in New Issue
Block a user