feat: implement Layer 2 per-file transit, RAM session, mock buffer, and tests (v0.2.0)

tests/test_app_integration.py

@@ -0,0 +1,115 @@
+import importlib.util
+import json
+from pathlib import Path
+import pytest
+
+app_path = Path(__file__).resolve().parent.parent / "site" / "app.py"
+spec = importlib.util.spec_from_file_location("site_app", app_path)
+site_app = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(site_app)
+
+app = site_app.app
+_mock_storage = site_app._mock_storage
+_mock_storage_lock = site_app._mock_storage_lock
+
+from upload.backend.session import cleanup, get_files
+import httpx
+
+
+@pytest.fixture
+def client():
+    app.config["TESTING"] = True
+    app.config["UPLOAD_HTTPX_TRANSPORT"] = httpx.WSGITransport(app=app)
+    with app.test_client() as client:
+        yield client
+
+
+def test_health(client):
+    res = client.get("/health")
+    assert res.status_code == 200
+    assert res.data == b"ok"
+
+
+def test_index_page(client):
+    res = client.get("/")
+    assert res.status_code == 200
+    assert b"upload-btn" in res.data
+    assert b"file-picker" in res.data
+
+
+def test_mock_buffer_crud(client):
+    key = "test_item_1"
+    # PUT
+    res = client.put(f"/mock-buffer/{key}", data=b"binary_payload_123")
+    assert res.status_code == 201
+
+    # GET
+    res = client.get(f"/mock-buffer/{key}")
+    assert res.status_code == 200
+    assert res.data == b"binary_payload_123"
+
+    # Status
+    res = client.get("/mock-buffer/status")
+    assert res.status_code == 200
+    data = res.get_json()
+    assert data["count"] >= 1
+    assert any(item["key"] == key for item in data["items"])
+
+    # DELETE
+    res = client.delete(f"/mock-buffer/{key}")
+    assert res.status_code == 204
+
+    # GET after delete -> 404
+    res = client.get(f"/mock-buffer/{key}")
+    assert res.status_code == 404
+
+
+def test_full_transit_flow_mock(client):
+    """Тестирует пофайловый транзит через mock-буфер и приём в сессию RAM."""
+    with _mock_storage_lock:
+        _mock_storage.clear()
+
+    # Шаг 1: Браузер кладёт файл в mock-буфер
+    key = "token_uuid_0"
+    file_bytes = b"%PDF-1.4 test document content for transit"
+    res = client.put(f"/mock-buffer/{key}", data=file_bytes)
+    assert res.status_code == 201
+
+    # Проверяем, что файл в буфере
+    with _mock_storage_lock:
+        assert key in _mock_storage
+
+    # Шаг 2: Браузер вызывает /api/upload_refs для этого файла
+    # Используем относительный URL пути /mock-buffer/key
+    res = client.post("/api/upload_refs", json={
+        "files": [
+            {"name": "contract.pdf", "size": len(file_bytes), "url": f"/mock-buffer/{key}"}
+        ]
+    })
+    assert res.status_code == 200
+    data = res.get_json()
+    assert data["ok"] is True
+    sid = data["session"]
+    assert data["count"] == 1
+    assert data["added"] == 1
+
+    # Шаг 3: Проверяем, что файл переместился в RAM сессии
+    files = get_files(sid)
+    assert len(files) == 1
+    assert files[0] == ("contract.pdf", file_bytes)
+
+    # Шаг 4: Проверяем, что файл удалился из mock-буфера (RAM буфера очищен)
+    with _mock_storage_lock:
+        assert key not in _mock_storage
+
+    # Шаг 5: Проверяем эндпоинт проверки сессии
+    res = client.get(f"/api/session/{sid}/files")
+    assert res.status_code == 200
+    s_data = res.get_json()
+    assert s_data["ok"] is True
+    assert s_data["session"] == sid
+    assert len(s_data["files"]) == 1
+    assert s_data["files"][0]["name"] == "contract.pdf"
+    assert s_data["files"][0]["size"] == len(file_bytes)
+
+    cleanup(sid)

tests/test_safe_name.py

@@ -0,0 +1,17 @@
+import pytest
+from upload.backend.upload_refs.safe_name import safe_name
+
+
+def test_safe_name_simple():
+    assert safe_name("test.txt") == "test.txt"
+    assert safe_name("folder/subfolder/file.pdf") == "folder/subfolder/file.pdf"
+    assert safe_name("folder\\subfolder\\file.pdf") == "folder/subfolder/file.pdf"
+
+
+def test_safe_name_traversal():
+    assert safe_name("../etc/passwd") == ""
+    assert safe_name("folder/../../etc/passwd") == ""
+    assert safe_name("/root/file.txt") == "root/file.txt"
+    assert safe_name("..") == ""
+    assert safe_name("") == ""
+    assert safe_name(None) == ""

tests/test_session.py

@@ -0,0 +1,61 @@
+import pytest
+from upload.backend.session import (
+    create_session, add_file, get_files, file_count,
+    store_result, get_result, store_csv, get_csv,
+    touch, pause_ttl, resume_ttl, request_cancel, get_cancel_event,
+    cleanup, configure, MAX_FILE_BYTES, MAX_SESSION_BYTES
+)
+from upload.backend.session.state import _sessions
+
+
+def test_session_lifecycle():
+    sid = create_session()
+    assert sid in _sessions
+    assert file_count(sid) == 0
+    assert get_files(sid) == []
+
+    # Add file
+    ok = add_file(sid, "doc.txt", b"hello world")
+    assert ok is True
+    assert file_count(sid) == 1
+    files = get_files(sid)
+    assert len(files) == 1
+    assert files[0] == ("doc.txt", b"hello world")
+
+    # Store result and csv
+    assert store_result(sid, b"PK...zip") is True
+    assert get_result(sid) == b"PK...zip"
+    assert store_csv(sid, "col1,col2\nval1,val2") is True
+    assert get_csv(sid) == "col1,col2\nval1,val2"
+
+    # Cancel event
+    ev = get_cancel_event(sid)
+    assert ev is not None
+    assert not ev.is_set()
+    assert request_cancel(sid) is True
+    assert ev.is_set()
+
+    # TTL methods do not raise
+    touch(sid)
+    pause_ttl(sid)
+    resume_ttl(sid)
+
+    # Cleanup
+    cleanup(sid)
+    assert sid not in _sessions
+    assert get_files(sid) is None
+    assert file_count(sid) == 0
+
+
+def test_session_limits():
+    sid = create_session()
+    configure(max_session_bytes=100)
+    try:
+        # Add 60 bytes - ok
+        assert add_file(sid, "f1.bin", b"x" * 60) is True
+        # Add 50 bytes - should fail (60 + 50 > 100)
+        assert add_file(sid, "f2.bin", b"x" * 50) is False
+        assert file_count(sid) == 1
+    finally:
+        configure(max_session_bytes=500 * 1024 * 1024)
+        cleanup(sid)

tests/test_upload_layer2.test.mjs

@@ -0,0 +1,233 @@
+// Юнит-тесты слоя 2 (фронт): putToVm + uploadViaVM с пофайловым транзитом.
+// Запуск: node --test tests/test_upload_layer2.test.mjs
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+// Полифилл File/Blob при необходимости
+if (typeof globalThis.File === 'undefined') {
+  globalThis.File = class File extends Blob {
+    constructor(parts, name, opts) {
+      super(parts, opts);
+      this.name = name;
+      this.lastModified = (opts && opts.lastModified) || Date.now();
+    }
+  };
+}
+
+import { putToVm } from '../upload/frontend/upload/put_to_vm.js';
+import { uploadViaVM } from '../upload/frontend/upload/upload_via_vm.js';
+
+const tick = () => new Promise(r => setTimeout(r, 0));
+
+// Мок XMLHttpRequest
+let lastXHR = null;
+class FakeXHR {
+  constructor() {
+    this.upload = {};
+    this.status = 0;
+    this.timeout = 0;
+    this.onload = null;
+    this.onerror = null;
+    this.ontimeout = null;
+    this.onabort = null;
+    this.method = null;
+    this.url = null;
+    this.body = null;
+    this.aborted = false;
+    lastXHR = this;
+  }
+  open(method, url) { this.method = method; this.url = url; }
+  send(body) { this.body = body; }
+  abort() {
+    this.aborted = true;
+    if (this.onabort) this.onabort();
+  }
+}
+globalThis.XMLHttpRequest = FakeXHR;
+
+// Мок fetch
+let fetchCalls = [];
+function installFetch(handler) {
+  fetchCalls = [];
+  globalThis.fetch = async (url, opts) => {
+    fetchCalls.push({ url, opts });
+    return handler(url, opts);
+  };
+}
+
+test('putToVm: успешная отправка', async () => {
+  const f = new File(['test-content'], 'doc.pdf');
+  const p = putToVm(f, 'https://vm-buffer/token_0');
+  lastXHR.status = 200;
+  lastXHR.onload();
+  await p;
+  assert.equal(lastXHR.method, 'PUT');
+  assert.equal(lastXHR.url, 'https://vm-buffer/token_0');
+  assert.equal(lastXHR.body, f);
+});
+
+test('putToVm: ошибка HTTP статуса', async () => {
+  const p = putToVm(new File(['abc'], 'doc.pdf'), 'https://vm-buffer/token_0');
+  lastXHR.status = 502;
+  lastXHR.statusText = 'Bad Gateway';
+  lastXHR.onload();
+  await assert.rejects(p, /HTTP 502/);
+});
+
+test('putToVm: сетевая ошибка', async () => {
+  const p = putToVm(new File(['abc'], 'doc.pdf'), 'https://vm-buffer/token_0');
+  lastXHR.onerror();
+  await assert.rejects(p, /Сетевая ошибка/);
+});
+
+test('putToVm: таймаут', async () => {
+  const p = putToVm(new File(['abc'], 'doc.pdf'), 'https://vm-buffer/token_0');
+  lastXHR.ontimeout();
+  await assert.rejects(p, /Таймаут/);
+});
+
+test('putToVm: прерывание через AbortSignal', async () => {
+  const ac = new AbortController();
+  const p = putToVm(new File(['abc'], 'doc.pdf'), 'https://vm-buffer/token_0', { signal: ac.signal });
+  ac.abort();
+  await assert.rejects(p, (err) => err.name === 'AbortError');
+  assert.equal(lastXHR.aborted, true);
+});
+
+test('uploadViaVM: пофайловый транзит (N файлов -> N PUT + N upload_refs)', async () => {
+  let callCount = 0;
+  installFetch(async (url, opts) => {
+    callCount++;
+    const body = JSON.parse(opts.body);
+    return {
+      ok: true,
+      json: async () => ({
+        ok: true,
+        session: body.session || 'created-sid-1',
+        count: callCount,
+      }),
+    };
+  });
+
+  const files = [
+    new File(['hello'], 'first.txt'),
+    new File(['world-data'], 'second.pdf'),
+  ];
+
+  const statuses = [];
+  const completed = [];
+
+  const p = uploadViaVM(files, {
+    vmUploadUrl: 'https://vm-buffer/upload/',
+    backendUploadUrl: '/api/upload_refs',
+    onFileStatus: (idx, status) => statuses.push({ idx, status }),
+    onFileComplete: (info) => completed.push(info),
+  });
+
+  // Файл 1: завершаем PUT
+  await tick();
+  assert.equal(lastXHR.method, 'PUT');
+  assert.ok(lastXHR.url.startsWith('https://vm-buffer/upload/'));
+  lastXHR.status = 201;
+  lastXHR.onload();
+
+  // Файл 2: завершаем PUT
+  await tick();
+  assert.equal(lastXHR.method, 'PUT');
+  assert.ok(lastXHR.url.startsWith('https://vm-buffer/upload/'));
+  lastXHR.status = 201;
+  lastXHR.onload();
+
+  const res = await p;
+  assert.equal(res.ok, true);
+  assert.equal(res.session, 'created-sid-1');
+  assert.equal(res.count, 2);
+
+  // Проверяем, что было РОВНО 2 вызова fetch (/api/upload_refs), по одному на каждый файл
+  assert.equal(fetchCalls.length, 2);
+
+  const call1Body = JSON.parse(fetchCalls[0].opts.body);
+  assert.equal(call1Body.files.length, 1);
+  assert.equal(call1Body.files[0].name, 'first.txt');
+
+  const call2Body = JSON.parse(fetchCalls[1].opts.body);
+  assert.equal(call2Body.files.length, 1);
+  assert.equal(call2Body.files[0].name, 'second.pdf');
+  // Сессия, созданная на 1 шаге, проброшена во 2 шаг
+  assert.equal(call2Body.session, 'created-sid-1');
+
+  assert.equal(completed.length, 2);
+  assert.equal(completed[0].name, 'first.txt');
+  assert.equal(completed[1].name, 'second.pdf');
+});
+
+test('uploadViaVM: поддержка формата FilePicker.getFiles() ({name, size, file, path})', async () => {
+  installFetch(async (url, opts) => {
+    return {
+      ok: true,
+      json: async () => ({ ok: true, session: 'sid-p', count: 1 }),
+    };
+  });
+
+  const pickerFiles = [
+    {
+      name: 'archive_doc.txt',
+      size: 42,
+      file: new File(['x'.repeat(42)], 'archive_doc.txt'),
+      path: 'folder/archive_doc.txt',
+    },
+  ];
+
+  const p = uploadViaVM(pickerFiles, {
+    vmUploadUrl: 'https://vm-buffer/upload/',
+  });
+
+  await tick();
+  lastXHR.status = 200;
+  lastXHR.onload();
+
+  const res = await p;
+  assert.equal(res.ok, true);
+  assert.equal(res.count, 1);
+
+  const body = JSON.parse(fetchCalls[0].opts.body);
+  assert.equal(body.files[0].name, 'archive_doc.txt');
+  assert.equal(body.files[0].size, 42);
+});
+
+test('uploadViaVM: обработка ошибки PUT', async () => {
+  installFetch(async () => ({ ok: true, json: async () => ({ ok: true }) }));
+
+  const files = [new File(['bad'], 'err.txt')];
+  const p = uploadViaVM(files, {
+    vmUploadUrl: 'https://vm-buffer/upload/',
+  });
+
+  await tick();
+  lastXHR.onerror();
+
+  const res = await p;
+  assert.equal(res.ok, false);
+  assert.ok(res.error.includes('Ошибка отправки файла'));
+  assert.equal(fetchCalls.length, 0); // POST не вызывался
+});
+
+test('uploadViaVM: прерывание через signal', async () => {
+  installFetch(async () => ({ ok: true, json: async () => ({ ok: true }) }));
+
+  const ac = new AbortController();
+  const files = [new File(['1'], 'f1.txt'), new File(['2'], 'f2.txt')];
+
+  const p = uploadViaVM(files, {
+    vmUploadUrl: 'https://vm-buffer/upload/',
+    signal: ac.signal,
+  });
+
+  await tick();
+  ac.abort();
+
+  const res = await p;
+  assert.equal(res.ok, false);
+  assert.equal(res.aborted, true);
+});

tests/test_upload_refs.py

@@ -0,0 +1,120 @@
+import threading
+from http.server import HTTPServer, BaseHTTPRequestHandler
+import pytest
+from flask import Flask
+from upload.backend.upload_refs.blueprint import create_upload_refs_blueprint
+from upload.backend.session import get_files, cleanup
+
+
+class MockBufferHandler(BaseHTTPRequestHandler):
+    storage = {}
+
+    def do_PUT(self):
+        length = int(self.headers.get("Content-Length", 0))
+        data = self.rfile.read(length)
+        MockBufferHandler.storage[self.path] = data
+        self.send_response(201)
+        self.end_headers()
+
+    def do_GET(self):
+        data = MockBufferHandler.storage.get(self.path)
+        if data is None:
+            self.send_response(404)
+            self.end_headers()
+            return
+        self.send_response(200)
+        self.send_header("Content-Length", str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_DELETE(self):
+        MockBufferHandler.storage.pop(self.path, None)
+        self.send_response(204)
+        self.end_headers()
+
+    def log_message(self, format, *args):
+        pass  # suppress console logs in tests
+
+
+@pytest.fixture(scope="module")
+def mock_server():
+    server = HTTPServer(("127.0.0.1", 0), MockBufferHandler)
+    port = server.server_port
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    yield f"http://127.0.0.1:{port}/buffer/"
+    server.shutdown()
+
+
+def test_upload_refs_pull_and_delete(mock_server):
+    # Put a file into mock buffer
+    file_path = "/buffer/token123_0"
+    content = b"Content of test document for layer 2 transit"
+    MockBufferHandler.storage[file_path] = content
+
+    received_events = []
+    def on_received(sid, name, data):
+        received_events.append((sid, name, data))
+
+    app = Flask(__name__)
+    bp = create_upload_refs_blueprint({
+        "vmUploadPrefix": mock_server,
+        "pullRetries": 1,
+        "pullRetryDelay": 0.1,
+        "onFileReceived": on_received,
+    })
+    app.register_blueprint(bp)
+    client = app.test_client()
+
+    file_url = mock_server + "token123_0"
+    resp = client.post("/api/upload_refs", json={
+        "files": [
+            {"name": "test_doc.pdf", "size": len(content), "url": file_url}
+        ]
+    })
+
+    assert resp.status_code == 200
+    data = resp.get_json()
+    assert data["ok"] is True
+    sid = data["session"]
+    assert data["count"] == 1
+    assert data["added"] == 1
+
+    # Verify file is in session memory
+    files = get_files(sid)
+    assert len(files) == 1
+    assert files[0] == ("test_doc.pdf", content)
+
+    # Verify callback for Layer 3 was triggered
+    assert len(received_events) == 1
+    assert received_events[0] == (sid, "test_doc.pdf", content)
+
+    # Verify file was DELETED from mock buffer (RAM clean!)
+    assert file_path not in MockBufferHandler.storage
+
+    cleanup(sid)
+
+
+def test_upload_refs_ssrf_protection(mock_server):
+    app = Flask(__name__)
+    bp = create_upload_refs_blueprint({
+        "vmUploadPrefix": mock_server,
+    })
+    app.register_blueprint(bp)
+    client = app.test_client()
+
+    # Try to pass an evil URL outside vmUploadPrefix
+    evil_url = "http://169.254.169.254/latest/meta-data/"
+    resp = client.post("/api/upload_refs", json={
+        "files": [
+            {"name": "evil.txt", "size": 100, "url": evil_url}
+        ]
+    })
+    assert resp.status_code == 200
+    data = resp.get_json()
+    assert data["ok"] is True
+    sid = data["session"]
+    assert data["added"] == 0
+    assert data["count"] == 0
+
+    cleanup(sid)