forcloud/upload-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove VM upload layer; keep file picker only

Browse Source
This commit is contained in:
“Naeel”
2026-09-05 09:23:13 +03:00
parent c0b87880ae
commit 85605fe8c6
20 changed files with 1 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files
site/app.py
+1 -9
View File
@@ -1,18 +1,15 @@
import json
import sys
from pathlib import Path
from flask import Flask, render_template, send_from_directory
ROOT = Path(__file__).resolve().parent.parent
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
with (ROOT / "config.json").open(encoding="utf-8") as config_file:
CONFIG = json.load(config_file)
VERSION = "0.1.2"
VERSION = "0.1.3"
app = Flask(__name__, template_folder="templates", static_folder="static")
@@ -32,10 +29,5 @@ def upload_frontend(filename):
return send_from_directory(ROOT / "upload" / "frontend", filename)
from routes.api_bp import register_api
register_api(app, CONFIG)
if __name__ == "__main__":
app.run(debug=False, host="0.0.0.0", port=5000)
site/routes/__init__.py
-1
View File
@@ -1 +0,0 @@
"""Маршруты демо-обёртки."""
site/routes/api_bp.py
-26
View File
@@ -1,26 +0,0 @@
"""Демо API: переиспользуемый upload_refs и заглушка обработки."""
from flask import Blueprint, jsonify, request
from upload.backend.session import get_files
from upload.backend.upload_refs import create_upload_refs_blueprint
def register_api(app, cfg):
app.register_blueprint(create_upload_refs_blueprint(cfg))
demo = Blueprint("demo_api", __name__, url_prefix=cfg.get("apiPrefix", "/api"))
@demo.post("/process")
def process():
data = request.get_json(silent=True) or {}
sid = data.get("session", "")
files = get_files(sid)
if files is None:
return jsonify({"ok": False, "error": "Session not found"}), 404
return jsonify({
"ok": True,
"session": sid,
"files": [{"name": name, "size": len(content)} for name, content in files],
})
app.register_blueprint(demo)
site/templates/index.html
-38
View File
@@ -18,7 +18,6 @@
<input id="folder-input" type="file" webkitdirectory directory multiple hidden>
<button id="files-btn" type="button">Выбрать файлы</button>
<button id="folder-btn" type="button" class="secondary">Выбрать папку</button>
<button id="upload-btn" type="button" disabled>Загрузить</button>
<button id="clear-btn" type="button" class="quiet">Очистить</button>
</section>
<p id="status" class="status" role="status"></p>
@@ -29,15 +28,10 @@
</table>
</div>
<p id="count" class="count"></p>
<section id="result" class="result" hidden>
<h2>Файлы в сессии</h2>
<ul id="result-list"></ul>
</section>
</main>
<script src="{{ url_for('static', filename='vendor/fflate.min.js') }}"></script>
<script type="module">
import { initUploadTable } from "/upload-frontend/table/init_upload_table.js";
import { uploadViaVM } from "/upload-frontend/upload/upload_via_vm.js";
const cfg = {{ config|tojson }};
const table = initUploadTable({
@@ -46,43 +40,11 @@
folderInputEl: document.querySelector('#folder-input'),
tableBodyEl: document.querySelector('#table-body'),
countEl: document.querySelector('#count'),
uploadBtnEl: document.querySelector('#upload-btn'),
});
const status = document.querySelector('#status');
document.querySelector('#files-btn').onclick = () => table.pickFiles();
document.querySelector('#folder-btn').onclick = () => table.pickFolder();
document.querySelector('#clear-btn').onclick = () => table.clear();
document.querySelector('#upload-btn').onclick = async () => {
const files = table.getFiles().map((item) => item.file);
table.setBusy(true);
status.textContent = 'Загрузка...';
const result = await uploadViaVM(files, cfg.vmUploadUrl, {
onUploadStatus: (text) => { status.textContent = text; },
onStatus: (name, text) => {
const item = table.getFiles().find((entry) => entry.name === name);
if (item) table.setStatus(item.path, text);
},
});
if (!result.ok) {
status.textContent = result.error;
table.setBusy(false);
return;
}
const response = await fetch('/api/process', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ session: result.session }),
});
const processed = await response.json();
status.textContent = `Готово: ${processed.files.length} файлов`;
document.querySelector('#result').hidden = false;
const resultList = document.querySelector('#result-list');
resultList.replaceChildren(...processed.files.map((file) => {
const item = document.createElement('li');
item.textContent = `${file.name} — ${file.size} B`;
return item;
}));
};
</script>
</body>
</html>
upload/backend/__init__.py
-1
View File
@@ -1 +0,0 @@
"""Backend-модули переиспользуемого слоя загрузки."""
upload/backend/session/__init__.py
-19
View File
@@ -1,19 +0,0 @@
"""Самодостаточное in-memory хранилище сессий."""
from .add_file import add_file
from .cleanup import cleanup
from .create_session import create_session
from .get_files import file_count, get_files
from .state import MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS, configure
__all__ = [
"create_session",
"add_file",
"get_files",
"file_count",
"cleanup",
"configure",
"TTL_SECONDS",
"MAX_FILE_BYTES",
"MAX_SESSION_BYTES",
]
upload/backend/session/add_file.py
-17
View File
@@ -1,17 +0,0 @@
"""Добавление файла в сессию."""
from . import state
def add_file(sid: str, filename: str, content: bytes) -> bool:
"""Добавить файл, если сессия существует и общий лимит не превышен."""
with state._lock:
session = state._sessions.get(sid)
if not session:
return False
total = sum(len(item_content) for _, item_content in session["files"])
if total + len(content) > state.MAX_SESSION_BYTES:
return False
session["files"].append((filename, content))
return True
upload/backend/session/cleanup.py
-12
View File
@@ -1,12 +0,0 @@
"""Удаление сессии."""
from .state import _lock, _sessions
def cleanup(sid: str):
"""Удалить сессию и остановить её TTL-таймер."""
with _lock:
session = _sessions.pop(sid, None)
if session and session.get("timer"):
session["timer"].cancel()
upload/backend/session/create_session.py
-19
View File
@@ -1,19 +0,0 @@
"""Создание сессии."""
import threading
import uuid
from .state import _lock, _sessions, _start_timer
def create_session() -> str:
"""Создать сессию и вернуть её уникальный идентификатор."""
sid = uuid.uuid4().hex
with _lock:
_sessions[sid] = {
"files": [],
"timer": _start_timer(sid),
"cancel": threading.Event(),
}
return sid
upload/backend/session/get_files.py
-21
View File
@@ -1,21 +0,0 @@
"""Чтение файлов сессии."""
from typing import List, Optional, Tuple
from .state import _lock, _sessions
def get_files(sid: str) -> Optional[List[Tuple[str, bytes]]]:
"""Вернуть файлы сессии или None, если сессия не найдена."""
with _lock:
session = _sessions.get(sid)
return list(session["files"]) if session else None
def file_count(sid: str) -> int:
"""Вернуть количество файлов в сессии."""
with _lock:
session = _sessions.get(sid)
return len(session["files"]) if session else 0
upload/backend/session/state.py
-37
View File
@@ -1,37 +0,0 @@
"""Общее состояние сессий, блокировка, лимиты и TTL."""
import threading
TTL_SECONDS = 30 * 60
MAX_FILE_BYTES = 50 * 1024 * 1024
MAX_SESSION_BYTES = 500 * 1024 * 1024
_sessions: dict = {}
_lock = threading.Lock()
def _start_timer(sid: str) -> threading.Timer:
"""Запустить таймер автоочистки сессии через TTL."""
def _clean():
with _lock:
_sessions.pop(sid, None)
timer = threading.Timer(TTL_SECONDS, _clean)
timer.daemon = True
timer.start()
return timer
def configure(max_file_bytes: int = None, max_session_bytes: int = None,
ttl_seconds: int = None):
"""Переопределить лимиты и TTL из конфигурации приложения."""
global MAX_FILE_BYTES, MAX_SESSION_BYTES, TTL_SECONDS
if max_file_bytes is not None:
MAX_FILE_BYTES = max_file_bytes
if max_session_bytes is not None:
MAX_SESSION_BYTES = max_session_bytes
if ttl_seconds is not None:
TTL_SECONDS = ttl_seconds
upload/backend/upload_refs/__init__.py
-15
View File
@@ -1,15 +0,0 @@
"""Переиспользуемый backend-слой закачки через ВМ."""
from .blueprint import create_upload_refs_blueprint
from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX
from .pull_file import pull_file
from .safe_name import safe_name
__all__ = [
"create_upload_refs_blueprint",
"safe_name",
"pull_file",
"PULL_RETRIES",
"PULL_RETRY_DELAY",
"VM_UPLOAD_PREFIX",
]
upload/backend/upload_refs/blueprint.py
-79
View File
@@ -1,79 +0,0 @@
"""Blueprint слоя закачки ссылок через ВМ-буфер."""
import logging
import requests
from flask import Blueprint, jsonify, request
from ..session import (MAX_FILE_BYTES, add_file, configure, create_session,
file_count, get_files)
from .config import PULL_RETRIES, PULL_RETRY_DELAY, VM_UPLOAD_PREFIX
from .pull_file import pull_file
from .safe_name import safe_name
log = logging.getLogger("upload.upload_refs")
def create_upload_refs_blueprint(cfg: dict) -> Blueprint:
"""Создать Blueprint с POST ``/upload_refs``."""
cfg = cfg or {}
prefix = cfg.get("apiPrefix", "/api")
vm_prefix = cfg.get("vmUploadUrl", VM_UPLOAD_PREFIX)
max_file_bytes = cfg.get("maxFileBytes", MAX_FILE_BYTES)
pull_retries = cfg.get("pullRetries", PULL_RETRIES)
pull_delay = cfg.get("pullRetryDelay", PULL_RETRY_DELAY)
pull_timeout = cfg.get("pullTimeout", 120)
configure(max_session_bytes=cfg.get("maxSessionBytes"))
blueprint = Blueprint("upload_refs", __name__, url_prefix=prefix)
@blueprint.route("/upload_refs", methods=["POST"])
def upload_refs():
data = request.get_json(silent=True) or {}
sid = data.get("session") or create_session()
refs = data.get("files") or []
if not isinstance(refs, list) or not refs:
return jsonify({"ok": False, "error": "No files"}), 400
added = 0
try:
with requests.Session() as client:
for ref in refs:
if not isinstance(ref, dict):
continue
name = safe_name(ref.get("name", ""))
url = ref.get("url")
if not name or not isinstance(url, str) or not url.startswith(vm_prefix):
continue
if (ref.get("size") or 0) > max_file_bytes:
_delete(client, url)
continue
content = pull_file(
client, url, max_file_bytes, pull_retries, pull_delay,
pull_timeout, sid=sid, name=name,
)
if not add_file(sid, name, content):
if get_files(sid) is None:
return jsonify({"ok": False, "error": "Session not found"}), 404
_delete(client, url)
continue
_delete(client, url)
added += 1
except Exception as error:
log.error("upload_refs: pull error sid=%s: %r", sid, error)
return jsonify({"ok": False, "error": "Pull failed: %s" % error}), 502
return jsonify({"ok": True, "session": sid, "count": file_count(sid)})
return blueprint
def _delete(client, url: str):
"""Удалить временный объект на ВМ, не ломая основной запрос при сбое."""
try:
client.delete(url, timeout=10)
except Exception:
log.warning("upload_refs: could not delete VM object url=%r", url)
upload/backend/upload_refs/config.py
-5
View File
@@ -1,5 +0,0 @@
"""Значения по умолчанию для pull из ВМ-буфера."""
PULL_RETRIES = 3
PULL_RETRY_DELAY = 2
VM_UPLOAD_PREFIX = "https://example.invalid/upload/"
upload/backend/upload_refs/pull_file.py
-38
View File
@@ -1,38 +0,0 @@
"""Исходящий pull файла из ВМ-буфера с ретраями и лимитом размера."""
import logging
import time
from .config import PULL_RETRIES, PULL_RETRY_DELAY
log = logging.getLogger("upload.upload_refs.pull")
def pull_file(client, url: str, max_bytes: int,
retries: int = PULL_RETRIES, delay: float = PULL_RETRY_DELAY,
timeout: int = 120, sid: str = None, name: str = None) -> bytes:
"""Забрать файл потоково, не принимая тело больше ``max_bytes``."""
last_error = None
for attempt in range(retries):
try:
response = client.get(url, stream=True, timeout=timeout)
response.raise_for_status()
chunks = []
total = 0
for chunk in response.iter_content(chunk_size=1024 * 1024):
if not chunk:
continue
total += len(chunk)
if total > max_bytes:
raise ValueError("file exceeds maxFileBytes")
chunks.append(chunk)
return b"".join(chunks)
except Exception as error:
last_error = error
log.warning("pull: attempt %d/%d failed sid=%s file=%r: %r",
attempt + 1, retries, sid, name, error)
if attempt + 1 < retries:
time.sleep(delay)
raise last_error if last_error else RuntimeError("pull failed")
upload/backend/upload_refs/safe_name.py
-13
View File
@@ -1,13 +0,0 @@
"""Санитизация имени файла с сохранением подпапок."""
def safe_name(name: str) -> str:
"""Вернуть безопасное относительное имя или пустую строку."""
if not isinstance(name, str) or not name:
return ""
parts = [part for part in name.replace("\\", "/").split("/")
if part and part != "."]
if not parts or any(part == ".." for part in parts):
return ""
return "/".join(parts)
upload/frontend/table/init_upload_table.js
-2
View File
@@ -10,7 +10,6 @@ export function initUploadTable(cfg) {
folderInputEl: cfg.folderInputEl,
tableBodyEl: cfg.tableBodyEl,
countEl: cfg.countEl,
uploadBtnEl: cfg.uploadBtnEl,
};
const state = { files: [], fileMeta: new Map(), overNames: new Set(), busy: false };
elements.fileInputEl.addEventListener('change', onFilesChange(state, cfg, elements));
@@ -36,7 +35,6 @@ export function initUploadTable(cfg) {
state.busy = busy;
elements.fileInputEl.disabled = busy;
elements.folderInputEl.disabled = busy;
elements.uploadBtnEl.disabled = busy || api.getFiles().length === 0;
},
};
api.render();
upload/frontend/table/render.js
-1
View File
@@ -18,5 +18,4 @@ export function render(state, elements) {
elements.countEl.textContent = `${included.length} учитываются`
+ (overCount ? ` + ${overCount} свыше лимита` : '')
+ ` · ${fs(size)}`;
elements.uploadBtnEl.disabled = included.length === 0;
}
upload/frontend/upload/put_to_vm.js
-19
View File
@@ -1,19 +0,0 @@
export function putToVm(file, url, options = {}) {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open('PUT', url);
xhr.timeout = options.timeoutMs || 300000;
xhr.upload.onprogress = (event) => {
if (event.lengthComputable && options.onProgress) {
options.onProgress(Math.round(event.loaded / event.total * 100));
}
};
xhr.onload = () => {
if (xhr.status >= 200 && xhr.status < 300) resolve();
else reject(new Error(`ВМ: HTTP ${xhr.status}`));
};
xhr.onerror = () => reject(new Error('Сеть (ВМ)'));
xhr.ontimeout = () => reject(new Error('Таймаут загрузки на ВМ'));
xhr.send(file);
});
}
upload/frontend/upload/upload_via_vm.js
-33
View File
@@ -1,33 +0,0 @@
import { putToVm } from './put_to_vm.js';
export async function uploadViaVM(files, vmUploadUrl, options = {}) {
const token = crypto.randomUUID();
const refs = [];
for (let index = 0; index < files.length; index += 1) {
const file = files[index];
const url = `${vmUploadUrl}${token}_${index}`;
options.onUploadStatus?.(`Загрузка на ВМ ${index + 1}/${files.length}: ${file.name}`);
try {
await putToVm(file, url, {
onProgress: (percent) => options.onStatus?.(file.name, `${percent}%`),
});
refs.push({ name: file.name, size: file.size, url });
} catch (error) {
options.onStatus?.(file.name, `Ошибка: ${error.message}`);
return { ok: false, error: `Ошибка загрузки на ВМ: ${error.message}` };
}
}
try {
const response = await fetch(`${options.apiBase || ''}/api/upload_refs`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ session: options.session || '', files: refs }),
});
const data = await response.json();
if (!response.ok || !data.ok) throw new Error(data.error || `HTTP ${response.status}`);
return { ok: true, session: data.session, count: data.count || refs.length };
} catch (error) {
return { ok: false, error: `Ошибка передачи ссылок: ${error.message}` };
}
}