Nail/contracts-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v2: чанки 30KB + MD5, /v2/chunk + /v2/finalize, Redis, worker, v2.0

Browse Source
This commit is contained in:
naeel
2026-06-17 23:14:34 +04:00
parent 562c27879e
commit c28baa446b
6 changed files with 416 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
site/app.py
+2
View File
@@ -14,6 +14,7 @@ import mimeutil
from test_routes import test_bp from test_routes import test_bp
from upload import upload_bp from upload import upload_bp
from api import api_bp from api import api_bp
from v2.chunk_api import bp as v2_bp
import redis_client import redis_client
import redis_worker import redis_worker
@@ -92,6 +93,7 @@ class ContractsApp:
self.app.register_blueprint(test_bp) self.app.register_blueprint(test_bp)
self.app.register_blueprint(upload_bp) self.app.register_blueprint(upload_bp)
self.app.register_blueprint(api_bp) self.app.register_blueprint(api_bp)
self.app.register_blueprint(v2_bp)
def _health(self): def _health(self):
return "OK", 200, {"Content-Type": "text/plain"} return "OK", 200, {"Content-Type": "text/plain"}
site/templates/upload.html
+31 -19
View File
@@ -5,6 +5,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Сверка договоров</title> <title>Сверка договоров</title>
<link rel="icon" type="image/png" href="{{ url_for('static', filename='favicon.png') }}"> <link rel="icon" type="image/png" href="{{ url_for('static', filename='favicon.png') }}">
<script src="https://cdnjs.cloudflare.com/ajax/libs/spark-md5/3.0.2/spark-md5.min.js"></script>
<script src="https://unpkg.com/lucide@latest"></script> <script src="https://unpkg.com/lucide@latest"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js"></script> <script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js"></script>
<style> <style>
@@ -59,7 +60,7 @@
<body> <body>
<div class="topbar"> <div class="topbar">
<img src="{{ url_for('static', filename='nubes-logo.svg') }}" alt="Nubes"> <img src="{{ url_for('static', filename='nubes-logo.svg') }}" alt="Nubes">
<span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.43</span></span> <span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v2.0</span></span>
</div> </div>
<div class="content"> <div class="content">
@@ -155,52 +156,63 @@
window.removeFile = removeFile; window.removeFile = removeFile;
// ── Чанковая загрузка 30KB ───────────────────────────────── // ── Чанковая загрузка v2 (30KB + MD5) ────────────────────
function uploadFileChunked(file, cid, onProgress) { function uploadFileChunked(file, cid, onProgress) {
return new Promise(function(resolve, reject) { return new Promise(function(resolve, reject) {
var reader = new FileReader(); var reader = new FileReader();
reader.onload = function() { reader.onload = function() {
var b64 = reader.result.split(',')[1]; var buffer = reader.result;
var bytes = new Uint8Array(buffer);
var b64 = btoa(String.fromCharCode.apply(null, bytes));
var checksum = SparkMD5.ArrayBuffer.hash(buffer);
var CHUNK = 30 * 1024; var CHUNK = 30 * 1024;
var totalChunks = Math.ceil(b64.length / CHUNK); var totalChunks = Math.ceil(b64.length / CHUNK);
var uploadId = file.name + '_' + Date.now() + '_' + Math.random().toString(36).substr(2, 6); var uploadId = 'u' + Date.now().toString(36) + Math.random().toString(36).substr(2, 7);
var done = 0; var done = 0;
function sendChunk(i) { function sendChunk(i) {
if (i >= totalChunks) { resolve({contract_id: cid}); return; } if (i >= totalChunks) {
var xhr = new XMLHttpRequest();
xhr.open('POST', '/v2/finalize');
xhr.setRequestHeader('Content-Type', 'application/json');
xhr.timeout = 15000;
xhr.onload = function() {
try {
var r = JSON.parse(xhr.responseText);
if (r.ok) resolve({contract_id: r.contract_id});
else reject(new Error(r.error));
} catch(e) { reject(new Error('Bad JSON')); }
};
xhr.onerror = function() { reject(new Error('Сеть')); };
xhr.ontimeout = function() { reject(new Error('Таймаут')); };
xhr.send(JSON.stringify({upload_id: uploadId, filename: file.name, total: totalChunks, checksum: checksum}));
return;
}
var piece = b64.substring(i * CHUNK, (i + 1) * CHUNK); var piece = b64.substring(i * CHUNK, (i + 1) * CHUNK);
var xhr = new XMLHttpRequest(); var xhr = new XMLHttpRequest();
xhr.open('POST', '/chunk_json'); xhr.open('POST', '/v2/chunk');
xhr.setRequestHeader('Content-Type', 'application/json'); xhr.setRequestHeader('Content-Type', 'application/json');
xhr.timeout = 10000; xhr.timeout = 10000;
xhr.onload = function() { xhr.onload = function() {
if (xhr.status === 200) { if (xhr.status === 200) {
try { try {
var resp = JSON.parse(xhr.responseText); var r = JSON.parse(xhr.responseText);
if (resp.error) { reject(new Error(resp.error)); return; } if (!r.ok) { reject(new Error(r.error)); return; }
if (resp.contract_id) cid = resp.contract_id;
done++; done++;
if (onProgress) onProgress(Math.round(done / totalChunks * 100)); if (onProgress) onProgress(Math.round(done / totalChunks * 100));
setTimeout(function() { sendChunk(i + 1); }, 100); setTimeout(function() { sendChunk(i + 1); }, 50);
} catch(e) { reject(new Error('Bad JSON')); } } catch(e) { reject(new Error('Bad JSON')); }
} else { reject(new Error('HTTP ' + xhr.status)); } } else { reject(new Error('HTTP ' + xhr.status)); }
}; };
xhr.onerror = function() { reject(new Error('Сеть')); }; xhr.onerror = function() { reject(new Error('Сеть')); };
xhr.ontimeout = function() { reject(new Error('Таймаут')); }; xhr.ontimeout = function() { reject(new Error('Таймаут')); };
xhr.send(JSON.stringify({ xhr.send(JSON.stringify({upload_id: uploadId, index: i, total: totalChunks, data: piece}));
upload_id: uploadId,
filename: file.name,
chunk_index: i,
total_chunks: totalChunks,
data: piece,
cid: cid || null
}));
} }
sendChunk(0); sendChunk(0);
}; };
reader.onerror = function() { reject(new Error('Read error')); }; reader.onerror = function() { reject(new Error('Read error')); };
reader.readAsDataURL(file); reader.readAsArrayBuffer(file);
}); });
} }
v2/chunk_api.py
+75
View File
@@ -0,0 +1,75 @@
"""v2/chunk_api.py — Приём чанков по 30KB, финализация."""
import sys, os, base64, hashlib, logging
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'site'))
import db, mimeutil
sys.path.insert(0, os.path.dirname(__file__))
import init as v2
from flask import Blueprint, request, jsonify
bp = Blueprint("v2", __name__, url_prefix="/v2")
@bp.route("/chunk", methods=["POST"])
def receive_chunk():
body = request.get_json(silent=True) or {}
if not body.get("upload_id") or not body.get("data"):
return jsonify({"ok": False, "error": "missing fields"}), 400
try:
v2.store_chunk(body["upload_id"], int(body.get("index", 0)), body["data"])
return jsonify({"ok": True, "received": int(body.get("index", 0))})
except Exception as e:
return jsonify({"ok": False, "error": str(e)}), 500
@bp.route("/finalize", methods=["POST"])
def finalize_upload():
body = request.get_json(silent=True) or {}
upload_id = body.get("upload_id", "")
filename = body.get("filename", "")
total = int(body.get("total", 0))
checksum = body.get("checksum", "")
if not all([upload_id, filename, total]):
return jsonify({"ok": False, "error": "missing fields"}), 400
chunks = v2.get_chunks(upload_id)
if len(chunks) != total:
return jsonify({"ok": False, "error": f"expected {total}, got {len(chunks)}"}), 409
try:
file_bytes = b"".join(base64.b64decode(c) for c in chunks)
except Exception as e:
return jsonify({"ok": False, "error": f"decode: {e}"}), 422
if checksum and hashlib.md5(file_bytes).hexdigest() != checksum:
return jsonify({"ok": False, "error": "checksum mismatch"}), 422
mime = mimeutil.guess_mime(filename) or "application/octet-stream"
# Сохранить в БД
conn, err = db.connect()
if err:
return jsonify({"ok": False, "error": f"db: {err}"}), 500
try:
cur = conn.cursor()
cur.execute(
"INSERT INTO contracts (number, client) VALUES (%s,%s) RETURNING id",
("б/н " + __import__("datetime").datetime.now().strftime("%Y%m%d-%H%M"), ""),
)
cid = cur.fetchone()[0]
cur.execute(
"INSERT INTO documents (filename, mime_type, original_bytes, status) VALUES (%s,%s,%s,'uploaded')",
(filename, mime, file_bytes),
)
cur.execute("SELECT id FROM documents WHERE filename=%s ORDER BY created_at DESC LIMIT 1", (filename,))
doc_id = cur.fetchone()[0]
cur.execute("INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')", (str(cid), str(doc_id)))
conn.commit()
cur.close()
finally:
db.put_conn(conn)
v2.delete_chunks(upload_id)
v2.push_finalize({"doc_id": str(doc_id), "filename": filename, "mime_type": mime})
return jsonify({"ok": True, "contract_id": str(cid), "doc_id": str(doc_id)})
v2/chunk_upload.js
+213
View File
@@ -0,0 +1,213 @@
/**
* v2/chunk_upload.js — Клиентская загрузка файла чанками по 32KB.
*
* Использование:
* <input type="file" id="file-input">
* <div id="upload-ui"></div>
* <script src="/static/v2/chunk_upload.js"></script>
*
* После успешной загрузки появляется кнопка «Парсинг».
*/
(function () {
"use strict";
const CHUNK_SIZE = 32 * 1024; // 32 KB
// ── MD5 (встроенная реализация без зависимостей) ──────────────────────────
// RFC 1321 — используется только для checksum, не для безопасности.
function md5(buffer) {
// SparkMD5 должен быть подключён отдельно, либо используем SubtleCrypto
// Здесь используем встроенный Web Crypto API (SHA-256 недостаточен — нужен MD5)
// Простая MD5 реализация:
return SparkMD5.ArrayBuffer.hash(buffer);
}
// ── UI ────────────────────────────────────────────────────────────────────
function getUI() {
return document.getElementById("upload-ui");
}
function setStatus(html) {
const ui = getUI();
if (ui) ui.innerHTML = html;
}
function renderProgress(filename, pct, elapsed) {
return `
<div class="upload-progress">
<strong>${escHtml(filename)}</strong><br>
<progress value="${pct}" max="100" style="width:100%"></progress>
<span>${pct}% &nbsp;|&nbsp; ${elapsed}с</span>
</div>`;
}
function escHtml(s) {
return String(s)
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;");
}
// ── XHR helper ────────────────────────────────────────────────────────────
function post(url, body) {
return new Promise(function (resolve, reject) {
const xhr = new XMLHttpRequest();
xhr.open("POST", url, true);
xhr.setRequestHeader("Content-Type", "application/json");
xhr.timeout = 15000;
xhr.onload = function () {
if (xhr.status >= 200 && xhr.status < 300) {
try { resolve(JSON.parse(xhr.responseText)); }
catch (e) { reject(new Error("bad json: " + xhr.responseText)); }
} else {
reject(new Error("HTTP " + xhr.status + ": " + xhr.responseText));
}
};
xhr.onerror = function () { reject(new Error("network error")); };
xhr.ontimeout = function () { reject(new Error("timeout")); };
xhr.send(JSON.stringify(body));
});
}
// ── Base64 encode ArrayBuffer ─────────────────────────────────────────────
function toBase64(buffer) {
let binary = "";
const bytes = new Uint8Array(buffer);
for (let i = 0; i < bytes.byteLength; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
}
// ── Генерация upload_id ───────────────────────────────────────────────────
function genId() {
return "u" + Date.now().toString(36) + Math.random().toString(36).slice(2, 7);
}
// ── Основная функция загрузки ─────────────────────────────────────────────
async function uploadFile(file) {
const uploadId = genId();
const total = Math.ceil(file.size / CHUNK_SIZE);
const startTs = Date.now();
// Считаем MD5 всего файла через SparkMD5
const fullBuffer = await file.arrayBuffer();
const checksum = md5(fullBuffer);
setStatus(renderProgress(file.name, 0, 0));
for (let i = 0; i < total; i++) {
const start = i * CHUNK_SIZE;
const end = Math.min(start + CHUNK_SIZE, file.size);
const slice = fullBuffer.slice(start, end);
const b64 = toBase64(slice);
const elapsed = Math.round((Date.now() - startTs) / 1000);
const pct = Math.round(((i + 1) / total) * 90); // до 90%, финал = 100%
setStatus(renderProgress(file.name, pct, elapsed));
let res;
try {
res = await post("/v2/chunk", {
upload_id: uploadId,
index: i,
total: total,
data: b64,
});
} catch (e) {
setStatus(`<p class="error">Ошибка чанка ${i}: ${escHtml(e.message)}</p>`);
return;
}
if (!res.ok) {
setStatus(`<p class="error">Сервер отклонил чанк ${i}: ${escHtml(res.error)}</p>`);
return;
}
}
// Финализация
const elapsed = Math.round((Date.now() - startTs) / 1000);
setStatus(renderProgress(file.name, 95, elapsed));
let fin;
try {
fin = await post("/v2/finalize", {
upload_id: uploadId,
filename: file.name,
mime_type: file.type || "application/octet-stream",
total: total,
checksum: checksum,
});
} catch (e) {
setStatus(`<p class="error">Ошибка финализации: ${escHtml(e.message)}</p>`);
return;
}
if (!fin.ok) {
setStatus(`<p class="error">Финализация не удалась: ${escHtml(fin.error)}</p>`);
return;
}
const docId = fin.doc_id;
const done = Math.round((Date.now() - startTs) / 1000);
setStatus(`
<div class="upload-done">
<strong>${escHtml(file.name)}</strong> загружен за ${done}с.<br>
<button id="btn-parse" data-docid="${escHtml(docId)}">Парсинг</button>
<div id="parse-status"></div>
</div>`);
document.getElementById("btn-parse").addEventListener("click", function () {
pollStatus(docId);
});
}
// ── Опрос статуса парсинга ────────────────────────────────────────────────
function pollStatus(docId) {
const ps = document.getElementById("parse-status");
if (ps) ps.textContent = "Парсинг запущен, ожидаем…";
let attempts = 0;
const timer = setInterval(async function () {
attempts++;
try {
const r = await fetch("/v2/status/" + docId);
const d = await r.json();
if (d.status === "parsed") {
clearInterval(timer);
if (ps) ps.innerHTML = `<span style="color:green">Готово: ${escHtml(d.filename)}</span>`;
} else if (d.status === "error") {
clearInterval(timer);
if (ps) ps.innerHTML = `<span style="color:red">Ошибка: ${escHtml(d.error || "unknown")}</span>`;
} else {
if (ps) ps.textContent = `Статус: ${d.status} (${attempts * 2}с)`;
}
} catch (e) {
if (ps) ps.textContent = `Ошибка опроса: ${e.message}`;
}
if (attempts > 60) { clearInterval(timer); }
}, 2000);
}
// ── Инициализация ─────────────────────────────────────────────────────────
document.addEventListener("DOMContentLoaded", function () {
const input = document.getElementById("file-input");
if (!input) return;
input.addEventListener("change", function () {
const file = input.files[0];
if (!file) return;
uploadFile(file).catch(function (e) {
setStatus(`<p class="error">Критическая ошибка: ${escHtml(e.message)}</p>`);
});
});
});
})();
v2/init.py
+44
View File
@@ -0,0 +1,44 @@
"""v2/init.py — Redis и хранилище чанков. БД — через site/db.py."""
import os, json, logging, redis
logger = logging.getLogger(__name__)
CHUNK_STORE_KEY = "v2:chunks:"
FINALIZE_QUEUE = "v2:finalize"
_r = None
def get_redis():
global _r
if _r is None:
_r = redis.Redis(
host=os.getenv("REDIS_HOST", "redisk8s.f6303a9d-4ab1-4b2f-8aa8-08f933d02f82.svc.cluster.local"),
port=int(os.getenv("REDIS_PORT", "6379")),
password=os.getenv("REDIS_PASS", "aLITloRefJEiCPqUc2xB"),
decode_responses=True,
socket_connect_timeout=5,
)
return _r
def store_chunk(upload_id, index, data_b64):
get_redis().rpush(f"{CHUNK_STORE_KEY}{upload_id}", json.dumps([index, data_b64]))
def get_chunks(upload_id):
raw = get_redis().lrange(f"{CHUNK_STORE_KEY}{upload_id}", 0, -1)
chunks = []
for r in raw:
idx, data = json.loads(r)
chunks.append((idx, data))
chunks.sort()
return [c[1] for c in chunks]
def delete_chunks(upload_id):
get_redis().delete(f"{CHUNK_STORE_KEY}{upload_id}")
def push_finalize(task):
get_redis().rpush(FINALIZE_QUEUE, json.dumps(task, ensure_ascii=False))
def pop_finalize(timeout=5):
result = get_redis().blpop(FINALIZE_QUEUE, timeout=timeout)
return json.loads(result[1]) if result else None
v2/worker.py
+51
View File
@@ -0,0 +1,51 @@
"""v2/worker.py — Фоновый парсинг через site/parser.py."""
import sys, os, json, logging, time
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'site'))
import db, parser as parser_mod, textify
sys.path.insert(0, os.path.dirname(__file__))
import init as v2
logging.basicConfig(level=logging.INFO, format="[worker] %(message)s")
logger = logging.getLogger(__name__)
def run():
logger.info("started")
while True:
try:
task = v2.pop_finalize(timeout=5)
if task:
_process(task)
except Exception as e:
logger.error("loop: %s", e)
time.sleep(1)
def _process(task):
doc_id = task.get("doc_id")
filename = task.get("filename", "")
mime = task.get("mime_type", "")
doc, err = db.query_one("SELECT original_bytes, mime_type FROM documents WHERE id=%s", (doc_id,))
if not doc:
logger.info("%s: not found", doc_id)
return
raw = doc["original_bytes"]
file_bytes = bytes(raw) if isinstance(raw, memoryview) else raw
mime = doc["mime_type"] or mime
pr = parser_mod.parse(file_bytes, mime)
elements = pr.get("elements", [])
if mime == "application/zip" and "files" in pr:
for zf in pr["files"]:
elements.extend(zf.get("elements", []))
text = textify.to_text(elements) if elements else ""
db.execute(
"UPDATE documents SET parsed_text=%s, elements_json=%s, status='parsed' WHERE id=%s",
(text, json.dumps(elements, ensure_ascii=False), doc_id),
)
logger.info("%s: parsed, %d elements", filename, len(elements))
if __name__ == "__main__":
run()