чанки в памяти (не БД), 30KB + 60s timeout, v1.43

This commit is contained in:
2026-06-17 23:00:04 +04:00
parent 92f929315a
commit 2525ad039f
2 changed files with 24 additions and 63 deletions
+22 -61
View File
@@ -20,9 +20,12 @@ import redis_worker
load_dotenv() load_dotenv()
import time as _time import time as _time
import re as _re
# Логи в памяти (быстро, не тормозит ответ) # Логи в памяти (быстро, не тормозит ответ)
_log_memory = [] _log_memory = []
# Хранилище чанков в памяти
_chunks_mem = {}
def _log(step, detail=""): def _log(step, detail=""):
"""Писать лог в память (мгновенно).""" """Писать лог в память (мгновенно)."""
@@ -190,10 +193,10 @@ class ContractsApp:
_log("upload_outer_error", str(e)) _log("upload_outer_error", str(e))
return jsonify({"error": f"Крах: {e}"}), 500 return jsonify({"error": f"Крах: {e}"}), 500
# ── Чанковая загрузка base64 (10KB) ─────────────────────── # ── Чанковая загрузка base64 (30KB) ───────────────────────
def _chunk_json(self): def _chunk_json(self):
"""Принять чанк base64. На последнем — собрать и сохранить.""" """Принять чанк base64 (в памяти). На последнем — в Redis."""
data = request.get_json(silent=True) or {} data = request.get_json(silent=True) or {}
upload_id = data.get("upload_id", "") upload_id = data.get("upload_id", "")
filename = data.get("filename", "") filename = data.get("filename", "")
@@ -205,68 +208,26 @@ class ContractsApp:
if not upload_id or not b64_piece: if not upload_id or not b64_piece:
return jsonify({"error": "Нет upload_id или data"}), 400 return jsonify({"error": "Нет upload_id или data"}), 400
# Храним в той же таблице chunks (chunk_data = base64 кусок как текст) if upload_id not in _chunks_mem:
db.execute( _chunks_mem[upload_id] = {"chunks": [None]*total_chunks, "filename": filename, "cid": cid, "received": 0}
"INSERT INTO chunks (upload_id, chunk_index, chunk_data, filename, mime, total_chunks, cid) "
"VALUES (%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (upload_id, chunk_index) DO NOTHING",
(upload_id, chunk_index, b64_piece.encode(), filename, "", total_chunks, cid),
)
count_res, _ = db.query("SELECT COUNT(*) FROM chunks WHERE upload_id=%s", (upload_id,)) store = _chunks_mem[upload_id]
received = count_res["rows"][0][0] if count_res else 0 if store["chunks"][chunk_index] is None:
store["chunks"][chunk_index] = b64_piece
store["received"] += 1
if received == total_chunks: if store["received"] == total_chunks:
try: full_b64 = "".join(store["chunks"])
rows_res, _ = db.query( fname = store["filename"]
"SELECT chunk_data FROM chunks WHERE upload_id=%s ORDER BY chunk_index", fcid = store["cid"]
(upload_id,), mime = mimeutil.guess_mime(fname) or "application/octet-stream"
) import threading
full_b64 = b"".join(r[0] for r in rows_res["rows"]).decode() task = {"filename": fname, "b64": full_b64, "mime": mime, "cid": fcid or ""}
threading.Thread(target=redis_client.push, args=(task,), daemon=True).start()
del _chunks_mem[upload_id]
return jsonify({"contract_id": fcid or "pending"})
# Получить метаданные return jsonify({"chunk": chunk_index, "received": store["received"], "total": total_chunks})
meta, _ = db.query_one("SELECT filename, cid FROM chunks WHERE upload_id=%s LIMIT 1", (upload_id,))
fname = meta["filename"] if meta else filename
fcid = meta["cid"] if meta else cid
conn, err = db.connect()
if err:
return jsonify({"error": f"БД: {err}"}), 500
try:
cur = conn.cursor()
if fcid:
contract_id = fcid
else:
cur.execute(
"INSERT INTO contracts (number, client) VALUES (%s, %s) RETURNING id",
("б" + __import__("datetime").datetime.now().strftime("%Y%m%d-%H%M"), ""),
)
contract_id = cur.fetchone()[0]
mime = mimeutil.guess_mime(fname) or "application/octet-stream"
cur.execute(
"INSERT INTO documents (filename, mime_type, original_b64, status) VALUES (%s,%s,%s,'uploaded')",
(fname, mime, full_b64),
)
cur.execute("SELECT id FROM documents WHERE filename=%s AND status='uploaded' ORDER BY created_at DESC LIMIT 1", (fname,))
row = cur.fetchone()
doc_id = row[0] if row else None
if doc_id:
cur.execute(
"INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')",
(str(contract_id), str(doc_id)),
)
conn.commit()
cur.close()
finally:
db.put_conn(conn)
except Exception as e:
return jsonify({"error": f"Сборка: {e}"}), 500
finally:
db.execute("DELETE FROM chunks WHERE upload_id=%s", (upload_id,))
return jsonify({"contract_id": str(contract_id)})
return jsonify({"chunk": chunk_index, "received": received, "total": total_chunks})
# ── Старая чанковая загрузка ─────────────────────────────── # ── Старая чанковая загрузка ───────────────────────────────
+2 -2
View File
@@ -59,7 +59,7 @@
<body> <body>
<div class="topbar"> <div class="topbar">
<img src="{{ url_for('static', filename='nubes-logo.svg') }}" alt="Nubes"> <img src="{{ url_for('static', filename='nubes-logo.svg') }}" alt="Nubes">
<span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.42</span></span> <span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.43</span></span>
</div> </div>
<div class="content"> <div class="content">
@@ -173,7 +173,7 @@
var xhr = new XMLHttpRequest(); var xhr = new XMLHttpRequest();
xhr.open('POST', '/chunk_json'); xhr.open('POST', '/chunk_json');
xhr.setRequestHeader('Content-Type', 'application/json'); xhr.setRequestHeader('Content-Type', 'application/json');
xhr.timeout = 15000; xhr.timeout = 60000;
xhr.onload = function() { xhr.onload = function() {
if (xhr.status === 200) { if (xhr.status === 200) {
try { try {