Implement reusable upload platform

upload/README.md

@@ -0,0 +1,13 @@
+# Upload module
+
+Скопируйте каталог `upload/` в проект и подключите:
+
+- `create_upload_refs_blueprint(config)` к Flask-приложению;
+- `initUploadTable(config)` из `frontend/table/init_upload_table.js`;
+- `uploadViaVM(files, vmUploadUrl)` из `frontend/upload/upload_via_vm.js`.
+
+В конфигурации задаются URL ВМ-буфера, допустимые расширения, лимиты и параметры
+повторных попыток pull. Backend-модуль хранит файлы сессии в памяти; обработка
+файлов остаётся ответственностью приложения, которое интегрирует этот модуль.
+
+Для ZIP перед ES-модулями загрузите локальный `fflate` из `site/static/vendor/`.

upload/__init__.py

@@ -0,0 +1 @@
+"""Переиспользуемые слои загрузки файлов."""

upload/backend/__init__.py

@@ -0,0 +1 @@
+"""Backend-модули переиспользуемого слоя загрузки."""

upload/backend/session/__init__.py

@@ -0,0 +1,19 @@
+"""Самодостаточное in-memory хранилище сессий."""
+
+from .add_file import add_file
+from .cleanup import cleanup
+from .create_session import create_session
+from .get_files import file_count, get_files
+from .state import MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS, configure
+
+__all__ = [
+    "create_session",
+    "add_file",
+    "get_files",
+    "file_count",
+    "cleanup",
+    "configure",
+    "TTL_SECONDS",
+    "MAX_FILE_BYTES",
+    "MAX_SESSION_BYTES",
+]

upload/backend/session/add_file.py

@@ -0,0 +1,17 @@
+"""Добавление файла в сессию."""
+
+from . import state
+
+
+def add_file(sid: str, filename: str, content: bytes) -> bool:
+    """Добавить файл, если сессия существует и общий лимит не превышен."""
+
+    with state._lock:
+        session = state._sessions.get(sid)
+        if not session:
+            return False
+        total = sum(len(item_content) for _, item_content in session["files"])
+        if total + len(content) > state.MAX_SESSION_BYTES:
+            return False
+        session["files"].append((filename, content))
+        return True

upload/backend/session/cleanup.py

@@ -0,0 +1,12 @@
+"""Удаление сессии."""
+
+from .state import _lock, _sessions
+
+
+def cleanup(sid: str):
+    """Удалить сессию и остановить её TTL-таймер."""
+
+    with _lock:
+        session = _sessions.pop(sid, None)
+        if session and session.get("timer"):
+            session["timer"].cancel()

upload/backend/session/create_session.py

@@ -0,0 +1,19 @@
+"""Создание сессии."""
+
+import threading
+import uuid
+
+from .state import _lock, _sessions, _start_timer
+
+
+def create_session() -> str:
+    """Создать сессию и вернуть её уникальный идентификатор."""
+
+    sid = uuid.uuid4().hex
+    with _lock:
+        _sessions[sid] = {
+            "files": [],
+            "timer": _start_timer(sid),
+            "cancel": threading.Event(),
+        }
+    return sid

upload/backend/session/get_files.py

@@ -0,0 +1,21 @@
+"""Чтение файлов сессии."""
+
+from typing import List, Optional, Tuple
+
+from .state import _lock, _sessions
+
+
+def get_files(sid: str) -> Optional[List[Tuple[str, bytes]]]:
+    """Вернуть файлы сессии или None, если сессия не найдена."""
+
+    with _lock:
+        session = _sessions.get(sid)
+        return list(session["files"]) if session else None
+
+
+def file_count(sid: str) -> int:
+    """Вернуть количество файлов в сессии."""
+
+    with _lock:
+        session = _sessions.get(sid)
+        return len(session["files"]) if session else 0

upload/backend/session/state.py

@@ -0,0 +1,37 @@
+"""Общее состояние сессий, блокировка, лимиты и TTL."""
+
+import threading
+
+
+TTL_SECONDS = 30 * 60
+MAX_FILE_BYTES = 50 * 1024 * 1024
+MAX_SESSION_BYTES = 500 * 1024 * 1024
+
+_sessions: dict = {}
+_lock = threading.Lock()
+
+
+def _start_timer(sid: str) -> threading.Timer:
+    """Запустить таймер автоочистки сессии через TTL."""
+
+    def _clean():
+        with _lock:
+            _sessions.pop(sid, None)
+
+    timer = threading.Timer(TTL_SECONDS, _clean)
+    timer.daemon = True
+    timer.start()
+    return timer
+
+
+def configure(max_file_bytes: int = None, max_session_bytes: int = None,
+              ttl_seconds: int = None):
+    """Переопределить лимиты и TTL из конфигурации приложения."""
+
+    global MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS
+    if max_file_bytes is not None:
+        MAX_FILE_BYTES = max_file_bytes
+    if max_session_bytes is not None:
+        MAX_SESSION_BYTES = max_session_bytes
+    if ttl_seconds is not None:
+        TTL_SECONDS = ttl_seconds

upload/backend/upload_refs/__init__.py

@@ -0,0 +1,15 @@
+"""Переиспользуемый backend-слой закачки через ВМ."""
+
+from .blueprint import create_upload_refs_blueprint
+from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX
+from .pull_file import pull_file
+from .safe_name import safe_name
+
+__all__ = [
+    "create_upload_refs_blueprint",
+    "safe_name",
+    "pull_file",
+    "PULL_RETRIES",
+    "PULL_RETRY_DELAY",
+    "VM_UPLOAD_PREFIX",
+]

upload/backend/upload_refs/blueprint.py

@@ -0,0 +1,79 @@
+"""Blueprint слоя закачки ссылок через ВМ-буфер."""
+
+import logging
+
+import requests
+from flask import Blueprint, jsonify, request
+
+from ..session import (MAX_FILE_BYTES, add_file, configure, create_session,
+                       file_count, get_files)
+from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX
+from .pull_file import pull_file
+from .safe_name import safe_name
+
+
+log = logging.getLogger("upload.upload_refs")
+
+
+def create_upload_refs_blueprint(cfg: dict) -> Blueprint:
+    """Создать Blueprint с POST ``/upload_refs``."""
+
+    cfg = cfg or {}
+    prefix = cfg.get("apiPrefix", "/api")
+    vm_prefix = cfg.get("vmUploadUrl", VM_UPLOAD_PREFIX)
+    max_file_bytes = cfg.get("maxFileBytes", MAX_FILE_BYTES)
+    pull_retries = cfg.get("pullRetries", PULL_RETRIES)
+    pull_delay = cfg.get("pullRetryDelay", PULL_RETRY_DELAY)
+    pull_timeout = cfg.get("pullTimeout", 120)
+    configure(max_session_bytes=cfg.get("maxSessionBytes"))
+
+    blueprint = Blueprint("upload_refs", __name__, url_prefix=prefix)
+
+    @blueprint.route("/upload_refs", methods=["POST"])
+    def upload_refs():
+        data = request.get_json(silent=True) or {}
+        sid = data.get("session") or create_session()
+        refs = data.get("files") or []
+        if not isinstance(refs, list) or not refs:
+            return jsonify({"ok": False, "error": "No files"}), 400
+
+        added = 0
+        try:
+            with requests.Session() as client:
+                for ref in refs:
+                    if not isinstance(ref, dict):
+                        continue
+                    name = safe_name(ref.get("name", ""))
+                    url = ref.get("url")
+                    if not name or not isinstance(url, str) or not url.startswith(vm_prefix):
+                        continue
+                    if (ref.get("size") or 0) > max_file_bytes:
+                        _delete(client, url)
+                        continue
+                    content = pull_file(
+                        client, url, max_file_bytes, pull_retries, pull_delay,
+                        pull_timeout, sid=sid, name=name,
+                    )
+                    if not add_file(sid, name, content):
+                        if get_files(sid) is None:
+                            return jsonify({"ok": False, "error": "Session not found"}), 404
+                        _delete(client, url)
+                        continue
+                    _delete(client, url)
+                    added += 1
+        except Exception as error:
+            log.error("upload_refs: pull error sid=%s: %r", sid, error)
+            return jsonify({"ok": False, "error": "Pull failed: %s" % error}), 502
+
+        return jsonify({"ok": True, "session": sid, "count": file_count(sid)})
+
+    return blueprint
+
+
+def _delete(client, url: str):
+    """Удалить временный объект на ВМ, не ломая основной запрос при сбое."""
+
+    try:
+        client.delete(url, timeout=10)
+    except Exception:
+        log.warning("upload_refs: could not delete VM object url=%r", url)

upload/backend/upload_refs/config.py

@@ -0,0 +1,5 @@
+"""Значения по умолчанию для pull из ВМ-буфера."""
+
+PULL_RETRIES = 3
+PULL_RETRY_DELAY = 2
+VM_UPLOAD_PREFIX = "https://example.invalid/upload/"

upload/backend/upload_refs/pull_file.py

@@ -0,0 +1,38 @@
+"""Исходящий pull файла из ВМ-буфера с ретраями и лимитом размера."""
+
+import logging
+import time
+
+from .config import PULL_RETRIES, PULL_RETRY_DELAY
+
+
+log = logging.getLogger("upload.upload_refs.pull")
+
+
+def pull_file(client, url: str, max_bytes: int,
+              retries: int = PULL_RETRIES, delay: float = PULL_RETRY_DELAY,
+              timeout: int = 120, sid: str = None, name: str = None) -> bytes:
+    """Забрать файл потоково, не принимая тело больше ``max_bytes``."""
+
+    last_error = None
+    for attempt in range(retries):
+        try:
+            response = client.get(url, stream=True, timeout=timeout)
+            response.raise_for_status()
+            chunks = []
+            total = 0
+            for chunk in response.iter_content(chunk_size=1024 * 1024):
+                if not chunk:
+                    continue
+                total += len(chunk)
+                if total > max_bytes:
+                    raise ValueError("file exceeds maxFileBytes")
+                chunks.append(chunk)
+            return b"".join(chunks)
+        except Exception as error:
+            last_error = error
+            log.warning("pull: attempt %d/%d failed sid=%s file=%r: %r",
+                        attempt + 1, retries, sid, name, error)
+            if attempt + 1 < retries:
+                time.sleep(delay)
+    raise last_error if last_error else RuntimeError("pull failed")

upload/backend/upload_refs/safe_name.py

@@ -0,0 +1,13 @@
+"""Санитизация имени файла с сохранением подпапок."""
+
+
+def safe_name(name: str) -> str:
+    """Вернуть безопасное относительное имя или пустую строку."""
+
+    if not isinstance(name, str) or not name:
+        return ""
+    parts = [part for part in name.replace("\\", "/").split("/")
+             if part and part != "."]
+    if not parts or any(part == ".." for part in parts):
+        return ""
+    return "/".join(parts)

upload/config.example.json

@@ -0,0 +1,9 @@
+{
+  "vmUploadUrl": "https://example.invalid/upload/",
+  "allowedExt": [".pdf", ".doc", ".docx", ".txt", ".md"],
+  "maxFileBytes": 52428800,
+  "maxSessionBytes": 524288000,
+  "apiPrefix": "/api",
+  "pullRetries": 3,
+  "pullRetryDelay": 2
+}

upload/frontend/table/add_file_with_dedup.js

@@ -0,0 +1,27 @@
+export function addFileWithDedup(state, file, cfg) {
+  const originalName = file.name;
+  const existing = state.fileMeta.get(originalName);
+  let name = originalName;
+
+  if (existing) {
+    if (existing.size === file.size) return false;
+    const dot = originalName.lastIndexOf('.');
+    const base = dot > 0 ? originalName.slice(0, dot) : originalName;
+    const extension = dot > 0 ? originalName.slice(dot) : '';
+    let suffix = 2;
+    while (state.fileMeta.has(`${base}_${suffix}${extension}`)) suffix += 1;
+    name = `${base}_${suffix}${extension}`;
+  }
+
+  const storedFile = new File([file], name, { lastModified: file.lastModified });
+  state.fileMeta.set(name, { size: storedFile.size });
+  const includedBytes = state.files.reduce(
+    (total, item) => total + (state.overNames.has(item.name) ? 0 : item.size), 0,
+  );
+  if (storedFile.size > cfg.maxFileBytes
+      || includedBytes + storedFile.size > cfg.maxSessionBytes) {
+    state.overNames.add(name);
+  }
+  state.files.push(storedFile);
+  return true;
+}

upload/frontend/table/esc.js

@@ -0,0 +1,9 @@
+export function esc(value) {
+  return String(value).replace(/[&<>"']/g, (character) => ({
+    '&': '&amp;',
+    '<': '&lt;',
+    '>': '&gt;',
+    '"': '&quot;',
+    "'": '&#39;',
+  }[character]));
+}

upload/frontend/table/fs.js

@@ -0,0 +1,5 @@
+export function fs(bytes) {
+  return bytes < 1024 ? `${bytes} B`
+    : bytes < 1048576 ? `${(bytes / 1024).toFixed(1)} KB`
+    : `${(bytes / 1048576).toFixed(1)} MB`;
+}

upload/frontend/table/init_upload_table.js

@@ -0,0 +1,44 @@
+import { addFiles } from './on_files_change.js';
+import { onFilesChange } from './on_files_change.js';
+import { onFolderChange } from './on_folder_change.js';
+import { render } from './render.js';
+import { setStatus } from './set_status.js';
+
+export function initUploadTable(cfg) {
+  const elements = {
+    fileInputEl: cfg.fileInputEl,
+    folderInputEl: cfg.folderInputEl,
+    tableBodyEl: cfg.tableBodyEl,
+    countEl: cfg.countEl,
+    uploadBtnEl: cfg.uploadBtnEl,
+  };
+  const state = { files: [], fileMeta: new Map(), overNames: new Set(), busy: false };
+  elements.fileInputEl.addEventListener('change', onFilesChange(state, cfg, elements));
+  elements.folderInputEl.addEventListener('change', onFolderChange(state, cfg, elements));
+
+  const api = {
+    pickFiles: () => elements.fileInputEl.click(),
+    pickFolder: () => elements.folderInputEl.click(),
+    addFiles: (files) => addFiles(state, cfg, files, elements),
+    getFiles: () => state.files
+      .filter((file) => !state.overNames.has(file.name))
+      .map((file) => ({ path: file.name, name: file.name, size: file.size, file })),
+    getOverNames: () => new Set(state.overNames),
+    setStatus: (path, html) => setStatus(path, html, state, elements),
+    render: () => render(state, elements),
+    clear: () => {
+      state.files = [];
+      state.fileMeta.clear();
+      state.overNames.clear();
+      render(state, elements);
+    },
+    setBusy: (busy) => {
+      state.busy = busy;
+      elements.fileInputEl.disabled = busy;
+      elements.folderInputEl.disabled = busy;
+      elements.uploadBtnEl.disabled = busy || api.getFiles().length === 0;
+    },
+  };
+  api.render();
+  return api;
+}

upload/frontend/table/on_files_change.js

@@ -0,0 +1,29 @@
+import { listZipFiles } from '../zip/list_zip_files.js';
+import { addFileWithDedup } from './add_file_with_dedup.js';
+import { render } from './render.js';
+
+export async function addFiles(state, cfg, files, elements) {
+  for (const file of Array.from(files)) {
+    if (!file.name.toLowerCase().endsWith('.zip')) {
+      addFileWithDedup(state, file, cfg);
+      continue;
+    }
+    try {
+      const extracted = await listZipFiles(file, cfg.allowedExt);
+      if (extracted.length) {
+        extracted.forEach((item) => addFileWithDedup(state, item, cfg));
+      } else {
+        addFileWithDedup(state, file, cfg);
+      }
+    } catch (error) {
+      addFileWithDedup(state, file, cfg);
+    }
+  }
+  render(state, elements);
+}
+
+export function onFilesChange(state, cfg, elements) {
+  return () => {
+    if (!state.busy) addFiles(state, cfg, elements.fileInputEl.files, elements);
+  };
+}

upload/frontend/table/on_folder_change.js

@@ -0,0 +1,37 @@
+import { listZipFiles } from '../zip/list_zip_files.js';
+import { addFileWithDedup } from './add_file_with_dedup.js';
+import { render } from './render.js';
+
+export function onFolderChange(state, cfg, elements) {
+  return async () => {
+    if (state.busy) return;
+    for (const file of Array.from(elements.folderInputEl.files)) {
+      const parts = (file.webkitRelativePath || file.name).split('/');
+      const relativePath = parts.slice(1).join('/') || file.name;
+      const lowerPath = relativePath.toLowerCase();
+      const directory = relativePath.includes('/')
+        ? relativePath.slice(0, relativePath.lastIndexOf('/')) : '';
+      if (lowerPath.endsWith('.zip')) {
+        try {
+          const extracted = await listZipFiles(file, cfg.allowedExt);
+          if (extracted.length) {
+            extracted.forEach((item) => addFileWithDedup(state, new File([item],
+              directory ? `${directory}/${item.name}` : item.name,
+              { lastModified: item.lastModified }), cfg));
+          } else {
+            addFileWithDedup(state, new File([file], relativePath,
+              { lastModified: file.lastModified }), cfg);
+          }
+        } catch (error) {
+          addFileWithDedup(state, new File([file], relativePath,
+            { lastModified: file.lastModified }), cfg);
+        }
+      } else if (cfg.allowedExt.some((extension) => lowerPath.endsWith(extension))) {
+        addFileWithDedup(state, new File([file], relativePath,
+          { lastModified: file.lastModified }), cfg);
+      }
+    }
+    elements.folderInputEl.value = '';
+    render(state, elements);
+  };
+}

upload/frontend/table/render.js

@@ -0,0 +1,22 @@
+import { esc } from './esc.js';
+import { fs } from './fs.js';
+
+export function render(state, elements) {
+  if (!state.files.length) {
+    elements.tableBodyEl.innerHTML = '<tr><td colspan="3">Нет выбранных файлов</td></tr>';
+  } else {
+    elements.tableBodyEl.innerHTML = state.files.map((file, index) => {
+      const over = state.overNames.has(file.name);
+      const status = over ? 'не учитывается (лимит)' : 'готов';
+      return `<tr id="row-${index}"><td>${esc(file.name)}</td>`
+        + `<td>${fs(file.size)}</td><td id="st-${index}">${status}</td></tr>`;
+    }).join('');
+  }
+  const included = state.files.filter((file) => !state.overNames.has(file.name));
+  const overCount = state.files.length - included.length;
+  const size = included.reduce((total, file) => total + file.size, 0);
+  elements.countEl.textContent = `${included.length} учитываются`
+    + (overCount ? ` + ${overCount} свыше лимита` : '')
+    + ` · ${fs(size)}`;
+  elements.uploadBtnEl.disabled = included.length === 0;
+}

upload/frontend/table/set_status.js

@@ -0,0 +1,5 @@
+export function setStatus(path, html, state, elements) {
+  const index = state.files.findIndex((file) => file.name === path);
+  const cell = elements.tableBodyEl.querySelector(`#st-${index}`);
+  if (cell) cell.innerHTML = html;
+}

upload/frontend/upload/put_to_vm.js

@@ -0,0 +1,19 @@
+export function putToVm(file, url, options = {}) {
+  return new Promise((resolve, reject) => {
+    const xhr = new XMLHttpRequest();
+    xhr.open('PUT', url);
+    xhr.timeout = options.timeoutMs || 300000;
+    xhr.upload.onprogress = (event) => {
+      if (event.lengthComputable && options.onProgress) {
+        options.onProgress(Math.round(event.loaded / event.total * 100));
+      }
+    };
+    xhr.onload = () => {
+      if (xhr.status >= 200 && xhr.status < 300) resolve();
+      else reject(new Error(`ВМ: HTTP ${xhr.status}`));
+    };
+    xhr.onerror = () => reject(new Error('Сеть (ВМ)'));
+    xhr.ontimeout = () => reject(new Error('Таймаут загрузки на ВМ'));
+    xhr.send(file);
+  });
+}

upload/frontend/upload/upload_via_vm.js

@@ -0,0 +1,33 @@
+import { putToVm } from './put_to_vm.js';
+
+export async function uploadViaVM(files, vmUploadUrl, options = {}) {
+  const token = crypto.randomUUID();
+  const refs = [];
+  for (let index = 0; index < files.length; index += 1) {
+    const file = files[index];
+    const url = `${vmUploadUrl}${token}_${index}`;
+    options.onUploadStatus?.(`Загрузка на ВМ ${index + 1}/${files.length}: ${file.name}`);
+    try {
+      await putToVm(file, url, {
+        onProgress: (percent) => options.onStatus?.(file.name, `${percent}%`),
+      });
+      refs.push({ name: file.name, size: file.size, url });
+    } catch (error) {
+      options.onStatus?.(file.name, `Ошибка: ${error.message}`);
+      return { ok: false, error: `Ошибка загрузки на ВМ: ${error.message}` };
+    }
+  }
+
+  try {
+    const response = await fetch(`${options.apiBase || ''}/api/upload_refs`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ session: options.session || '', files: refs }),
+    });
+    const data = await response.json();
+    if (!response.ok || !data.ok) throw new Error(data.error || `HTTP ${response.status}`);
+    return { ok: true, session: data.session, count: data.count || refs.length };
+  } catch (error) {
+    return { ok: false, error: `Ошибка передачи ссылок: ${error.message}` };
+  }
+}

upload/frontend/zip/list_zip_files.js

@@ -0,0 +1,32 @@
+// Рекурсивно получить из ZIP только файлы с разрешёнными расширениями.
+
+function extensionAllowed(name, allowedExt) {
+  const lowerName = name.toLowerCase();
+  return allowedExt.some((extension) => lowerName.endsWith(extension.toLowerCase()));
+}
+
+function makeFile(data, name) {
+  return new File([data], name);
+}
+
+async function listEntries(data, prefix, allowedExt, depth) {
+  if (depth > 20) throw new Error('Слишком глубокая вложенность ZIP');
+  const entries = fflate.unzipSync(data);
+  const files = [];
+
+  for (const [entryName, entryData] of Object.entries(entries)) {
+    if (entryName.endsWith('/')) continue;
+    const path = prefix ? `${prefix}/${entryName}` : entryName;
+    if (entryName.toLowerCase().endsWith('.zip')) {
+      files.push(...await listEntries(entryData, path, allowedExt, depth + 1));
+    } else if (extensionAllowed(entryName, allowedExt)) {
+      files.push(makeFile(entryData, path));
+    }
+  }
+  return files;
+}
+
+export async function listZipFiles(file, allowedExt) {
+  const data = new Uint8Array(await file.arrayBuffer());
+  return listEntries(data, '', allowedExt, 0);
+}