чанки в памяти (не БД), 30KB + 60s timeout, v1.43
This commit is contained in:
+21
-60
@@ -20,9 +20,12 @@ import redis_worker
|
|||||||
load_dotenv()
|
load_dotenv()
|
||||||
|
|
||||||
import time as _time
|
import time as _time
|
||||||
|
import re as _re
|
||||||
|
|
||||||
# Логи в памяти (быстро, не тормозит ответ)
|
# Логи в памяти (быстро, не тормозит ответ)
|
||||||
_log_memory = []
|
_log_memory = []
|
||||||
|
# Хранилище чанков в памяти
|
||||||
|
_chunks_mem = {}
|
||||||
|
|
||||||
def _log(step, detail=""):
|
def _log(step, detail=""):
|
||||||
"""Писать лог в память (мгновенно)."""
|
"""Писать лог в память (мгновенно)."""
|
||||||
@@ -190,10 +193,10 @@ class ContractsApp:
|
|||||||
_log("upload_outer_error", str(e))
|
_log("upload_outer_error", str(e))
|
||||||
return jsonify({"error": f"Крах: {e}"}), 500
|
return jsonify({"error": f"Крах: {e}"}), 500
|
||||||
|
|
||||||
# ── Чанковая загрузка base64 (10KB) ───────────────────────
|
# ── Чанковая загрузка base64 (30KB) ───────────────────────
|
||||||
|
|
||||||
def _chunk_json(self):
|
def _chunk_json(self):
|
||||||
"""Принять чанк base64. На последнем — собрать и сохранить."""
|
"""Принять чанк base64 (в памяти). На последнем — в Redis."""
|
||||||
data = request.get_json(silent=True) or {}
|
data = request.get_json(silent=True) or {}
|
||||||
upload_id = data.get("upload_id", "")
|
upload_id = data.get("upload_id", "")
|
||||||
filename = data.get("filename", "")
|
filename = data.get("filename", "")
|
||||||
@@ -205,68 +208,26 @@ class ContractsApp:
|
|||||||
if not upload_id or not b64_piece:
|
if not upload_id or not b64_piece:
|
||||||
return jsonify({"error": "Нет upload_id или data"}), 400
|
return jsonify({"error": "Нет upload_id или data"}), 400
|
||||||
|
|
||||||
# Храним в той же таблице chunks (chunk_data = base64 кусок как текст)
|
if upload_id not in _chunks_mem:
|
||||||
db.execute(
|
_chunks_mem[upload_id] = {"chunks": [None]*total_chunks, "filename": filename, "cid": cid, "received": 0}
|
||||||
"INSERT INTO chunks (upload_id, chunk_index, chunk_data, filename, mime, total_chunks, cid) "
|
|
||||||
"VALUES (%s,%s,%s,%s,%s,%s,%s) ON CONFLICT (upload_id, chunk_index) DO NOTHING",
|
|
||||||
(upload_id, chunk_index, b64_piece.encode(), filename, "", total_chunks, cid),
|
|
||||||
)
|
|
||||||
|
|
||||||
count_res, _ = db.query("SELECT COUNT(*) FROM chunks WHERE upload_id=%s", (upload_id,))
|
store = _chunks_mem[upload_id]
|
||||||
received = count_res["rows"][0][0] if count_res else 0
|
if store["chunks"][chunk_index] is None:
|
||||||
|
store["chunks"][chunk_index] = b64_piece
|
||||||
if received == total_chunks:
|
store["received"] += 1
|
||||||
try:
|
|
||||||
rows_res, _ = db.query(
|
|
||||||
"SELECT chunk_data FROM chunks WHERE upload_id=%s ORDER BY chunk_index",
|
|
||||||
(upload_id,),
|
|
||||||
)
|
|
||||||
full_b64 = b"".join(r[0] for r in rows_res["rows"]).decode()
|
|
||||||
|
|
||||||
# Получить метаданные
|
|
||||||
meta, _ = db.query_one("SELECT filename, cid FROM chunks WHERE upload_id=%s LIMIT 1", (upload_id,))
|
|
||||||
fname = meta["filename"] if meta else filename
|
|
||||||
fcid = meta["cid"] if meta else cid
|
|
||||||
|
|
||||||
conn, err = db.connect()
|
|
||||||
if err:
|
|
||||||
return jsonify({"error": f"БД: {err}"}), 500
|
|
||||||
try:
|
|
||||||
cur = conn.cursor()
|
|
||||||
if fcid:
|
|
||||||
contract_id = fcid
|
|
||||||
else:
|
|
||||||
cur.execute(
|
|
||||||
"INSERT INTO contracts (number, client) VALUES (%s, %s) RETURNING id",
|
|
||||||
("б/н " + __import__("datetime").datetime.now().strftime("%Y%m%d-%H%M"), ""),
|
|
||||||
)
|
|
||||||
contract_id = cur.fetchone()[0]
|
|
||||||
|
|
||||||
|
if store["received"] == total_chunks:
|
||||||
|
full_b64 = "".join(store["chunks"])
|
||||||
|
fname = store["filename"]
|
||||||
|
fcid = store["cid"]
|
||||||
mime = mimeutil.guess_mime(fname) or "application/octet-stream"
|
mime = mimeutil.guess_mime(fname) or "application/octet-stream"
|
||||||
cur.execute(
|
import threading
|
||||||
"INSERT INTO documents (filename, mime_type, original_b64, status) VALUES (%s,%s,%s,'uploaded')",
|
task = {"filename": fname, "b64": full_b64, "mime": mime, "cid": fcid or ""}
|
||||||
(fname, mime, full_b64),
|
threading.Thread(target=redis_client.push, args=(task,), daemon=True).start()
|
||||||
)
|
del _chunks_mem[upload_id]
|
||||||
cur.execute("SELECT id FROM documents WHERE filename=%s AND status='uploaded' ORDER BY created_at DESC LIMIT 1", (fname,))
|
return jsonify({"contract_id": fcid or "pending"})
|
||||||
row = cur.fetchone()
|
|
||||||
doc_id = row[0] if row else None
|
|
||||||
if doc_id:
|
|
||||||
cur.execute(
|
|
||||||
"INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')",
|
|
||||||
(str(contract_id), str(doc_id)),
|
|
||||||
)
|
|
||||||
conn.commit()
|
|
||||||
cur.close()
|
|
||||||
finally:
|
|
||||||
db.put_conn(conn)
|
|
||||||
except Exception as e:
|
|
||||||
return jsonify({"error": f"Сборка: {e}"}), 500
|
|
||||||
finally:
|
|
||||||
db.execute("DELETE FROM chunks WHERE upload_id=%s", (upload_id,))
|
|
||||||
|
|
||||||
return jsonify({"contract_id": str(contract_id)})
|
return jsonify({"chunk": chunk_index, "received": store["received"], "total": total_chunks})
|
||||||
|
|
||||||
return jsonify({"chunk": chunk_index, "received": received, "total": total_chunks})
|
|
||||||
|
|
||||||
# ── Старая чанковая загрузка ───────────────────────────────
|
# ── Старая чанковая загрузка ───────────────────────────────
|
||||||
|
|
||||||
|
|||||||
@@ -59,7 +59,7 @@
|
|||||||
<body>
|
<body>
|
||||||
<div class="topbar">
|
<div class="topbar">
|
||||||
<img src="{{ url_for('static', filename='nubes-logo.svg') }}" alt="Nubes">
|
<img src="{{ url_for('static', filename='nubes-logo.svg') }}" alt="Nubes">
|
||||||
<span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.42</span></span>
|
<span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.43</span></span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="content">
|
<div class="content">
|
||||||
@@ -173,7 +173,7 @@
|
|||||||
var xhr = new XMLHttpRequest();
|
var xhr = new XMLHttpRequest();
|
||||||
xhr.open('POST', '/chunk_json');
|
xhr.open('POST', '/chunk_json');
|
||||||
xhr.setRequestHeader('Content-Type', 'application/json');
|
xhr.setRequestHeader('Content-Type', 'application/json');
|
||||||
xhr.timeout = 15000;
|
xhr.timeout = 60000;
|
||||||
xhr.onload = function() {
|
xhr.onload = function() {
|
||||||
if (xhr.status === 200) {
|
if (xhr.status === 200) {
|
||||||
try {
|
try {
|
||||||
|
|||||||
Reference in New Issue
Block a user