чанковая загрузка 50KB, /chunk endpoint, v1.15
This commit is contained in:
+85
-16
@@ -17,6 +17,28 @@ from api import api_bp
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Хранилище чанков (в памяти, теряется при рестарте)
|
||||
_chunk_store = {}
|
||||
|
||||
|
||||
def _save_file_to_db(filename, file_bytes, mime, contract_id):
|
||||
"""Сохранить один файл в БД: documents + supplements. Возвращает doc_id или None."""
|
||||
db.execute(
|
||||
"INSERT INTO documents (filename, mime_type, original_bytes, status) VALUES (%s,%s,%s,'uploaded')",
|
||||
(filename, mime, file_bytes),
|
||||
)
|
||||
doc, _ = db.query_one(
|
||||
"SELECT id FROM documents WHERE filename=%s AND status='uploaded' ORDER BY created_at DESC LIMIT 1",
|
||||
(filename,),
|
||||
)
|
||||
doc_id = doc["id"] if doc else None
|
||||
if doc_id:
|
||||
db.execute(
|
||||
"INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')",
|
||||
(str(contract_id), str(doc_id)),
|
||||
)
|
||||
return doc_id
|
||||
|
||||
|
||||
class ContractsApp:
|
||||
def __init__(self):
|
||||
@@ -27,6 +49,7 @@ class ContractsApp:
|
||||
def _add_routes(self):
|
||||
self.app.add_url_rule("/", "index", self._index, methods=["GET", "POST"])
|
||||
self.app.add_url_rule("/parse/<cid>", "parse", self._parse, methods=["GET"])
|
||||
self.app.add_url_rule("/chunk", "chunk", self._chunk_upload, methods=["POST"])
|
||||
self.app.add_url_rule("/health", "health", self._health)
|
||||
self.app.register_blueprint(test_bp)
|
||||
self.app.register_blueprint(upload_bp)
|
||||
@@ -74,25 +97,71 @@ class ContractsApp:
|
||||
continue
|
||||
filename = f.filename
|
||||
mime = mimeutil.guess_mime(filename) or f.content_type or "application/octet-stream"
|
||||
file_bytes = f.read()
|
||||
|
||||
db.execute(
|
||||
"INSERT INTO documents (filename, mime_type, original_bytes, status) VALUES (%s,%s,%s,'uploaded')",
|
||||
(filename, mime, file_bytes),
|
||||
)
|
||||
doc, _ = db.query_one(
|
||||
"SELECT id FROM documents WHERE filename=%s AND status='uploaded' ORDER BY created_at DESC LIMIT 1",
|
||||
(filename,),
|
||||
)
|
||||
doc_id = doc["id"] if doc else None
|
||||
if doc_id:
|
||||
db.execute(
|
||||
"INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')",
|
||||
(str(contract_id), str(doc_id)),
|
||||
)
|
||||
_save_file_to_db(filename, f.read(), mime, str(contract_id))
|
||||
|
||||
return jsonify({"contract_id": str(contract_id)})
|
||||
|
||||
# ── Чанковая загрузка ──────────────────────────────────────
|
||||
|
||||
def _chunk_upload(self):
|
||||
"""Принять чанк файла. На последнем — собрать и сохранить."""
|
||||
upload_id = request.form.get("upload_id", "")
|
||||
filename = request.form.get("filename", "")
|
||||
chunk_index = int(request.form.get("chunk_index", 0))
|
||||
total_chunks = int(request.form.get("total_chunks", 1))
|
||||
cid = request.form.get("cid")
|
||||
chunk_file = request.files.get("chunk")
|
||||
if not chunk_file:
|
||||
return jsonify({"error": "Нет чанка"}), 400
|
||||
|
||||
chunk_data = chunk_file.read()
|
||||
|
||||
if upload_id not in _chunk_store:
|
||||
_chunk_store[upload_id] = {
|
||||
"filename": filename,
|
||||
"mime": mimeutil.guess_mime(filename) or "application/octet-stream",
|
||||
"chunks": [None] * total_chunks,
|
||||
"total": total_chunks,
|
||||
"cid": cid,
|
||||
"received": 0,
|
||||
}
|
||||
|
||||
store = _chunk_store[upload_id]
|
||||
if store["chunks"][chunk_index] is None:
|
||||
store["chunks"][chunk_index] = chunk_data
|
||||
store["received"] += 1
|
||||
|
||||
if store["received"] == total_chunks:
|
||||
# Собрать файл
|
||||
file_bytes = b"".join(store["chunks"])
|
||||
mime = store["mime"]
|
||||
fname = store["filename"]
|
||||
cid = store["cid"]
|
||||
|
||||
# Создать контракт если нет
|
||||
if cid:
|
||||
contract_id = cid
|
||||
else:
|
||||
conn, err = db.connect()
|
||||
if err:
|
||||
del _chunk_store[upload_id]
|
||||
return jsonify({"error": f"БД: {err}"}), 500
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"INSERT INTO contracts (number, client) VALUES (%s, %s) RETURNING id",
|
||||
("б/н " + __import__("datetime").datetime.now().strftime("%Y%m%d-%H%M"), ""),
|
||||
)
|
||||
contract_id = cur.fetchone()[0]
|
||||
conn.commit()
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
_save_file_to_db(fname, file_bytes, mime, str(contract_id))
|
||||
del _chunk_store[upload_id]
|
||||
return jsonify({"contract_id": str(contract_id)})
|
||||
|
||||
return jsonify({"chunk": chunk_index, "received": store["received"], "total": total_chunks})
|
||||
|
||||
def _show_contract(self, cid):
|
||||
"""Показать договор с кнопкой «Обработать»."""
|
||||
contract, _ = db.query_one("SELECT id,number,created_at FROM contracts WHERE id=%s", (cid,))
|
||||
|
||||
+48
-29
@@ -59,7 +59,7 @@
|
||||
<body>
|
||||
<div class="topbar">
|
||||
<img src="{{ url_for('static', filename='nubes-logo.svg') }}" alt="Nubes">
|
||||
<span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.14</span></span>
|
||||
<span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.15</span></span>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
@@ -155,37 +155,51 @@
|
||||
|
||||
window.removeFile = removeFile;
|
||||
|
||||
// ── Загрузка файла (XHR, возвращает Promise только при успехе/ошибке) ──
|
||||
// ── Чанковая загрузка (50KB) ────────────────────────────────
|
||||
|
||||
function uploadFileXHR(file, cid, onProgress) {
|
||||
return new Promise(function(resolve, reject) {
|
||||
var xhr = new XMLHttpRequest();
|
||||
var url = '/' + (cid ? '?cid=' + cid : '');
|
||||
xhr.open('POST', url);
|
||||
xhr.timeout = 120000;
|
||||
async function uploadFileChunked(file, cid, onProgress) {
|
||||
var CHUNK_SIZE = 50 * 1024;
|
||||
var totalChunks = Math.ceil(file.size / CHUNK_SIZE);
|
||||
var uploadId = file.name + '_' + Date.now() + '_' + Math.random().toString(36).substr(2, 6);
|
||||
var totalSent = 0;
|
||||
var lastResp = null;
|
||||
|
||||
xhr.upload.onprogress = function(e) {
|
||||
if (e.lengthComputable && onProgress) {
|
||||
onProgress(Math.round(e.loaded / e.total * 100));
|
||||
}
|
||||
};
|
||||
for (var i = 0; i < totalChunks; i++) {
|
||||
var start = i * CHUNK_SIZE;
|
||||
var end = Math.min(start + CHUNK_SIZE, file.size);
|
||||
var chunk = file.slice(start, end);
|
||||
|
||||
xhr.onload = function() {
|
||||
if (xhr.status === 200) {
|
||||
try { resolve(JSON.parse(xhr.responseText)); }
|
||||
catch(e) { reject(new Error('Bad JSON')); }
|
||||
} else {
|
||||
reject(new Error('HTTP ' + xhr.status));
|
||||
}
|
||||
};
|
||||
var resp = await new Promise(function(resolve, reject) {
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', '/chunk');
|
||||
xhr.timeout = 60000;
|
||||
|
||||
xhr.onerror = function() { reject(new Error('Сеть')); };
|
||||
xhr.ontimeout = function() { reject(new Error('Таймаут')); };
|
||||
xhr.onload = function() {
|
||||
if (xhr.status === 200) {
|
||||
try { resolve(JSON.parse(xhr.responseText)); }
|
||||
catch(e) { reject(new Error('Bad JSON')); }
|
||||
} else {
|
||||
reject(new Error('HTTP ' + xhr.status));
|
||||
}
|
||||
};
|
||||
xhr.onerror = function() { reject(new Error('Сеть')); };
|
||||
xhr.ontimeout = function() { reject(new Error('Таймаут')); };
|
||||
|
||||
var fd = new FormData();
|
||||
fd.append('files', file);
|
||||
xhr.send(fd);
|
||||
});
|
||||
var fd = new FormData();
|
||||
fd.append('chunk', chunk);
|
||||
fd.append('upload_id', uploadId);
|
||||
fd.append('filename', file.name);
|
||||
fd.append('chunk_index', i);
|
||||
fd.append('total_chunks', totalChunks);
|
||||
if (cid) fd.append('cid', cid);
|
||||
xhr.send(fd);
|
||||
});
|
||||
|
||||
if (resp && resp.contract_id) lastResp = resp;
|
||||
totalSent += (end - start);
|
||||
if (onProgress) onProgress(Math.round(totalSent / file.size * 100));
|
||||
}
|
||||
return lastResp;
|
||||
}
|
||||
|
||||
// ── Выбор файла → сразу загрузка ────────────────────────────
|
||||
@@ -209,11 +223,16 @@
|
||||
var startTime = Date.now();
|
||||
|
||||
try {
|
||||
var resp = await uploadFileXHR(f, contractId, function(pct) {
|
||||
var resp = await uploadFileChunked(f, contractId, function(pct) {
|
||||
fileQueue[rowIdx].status = '<span style="color:var(--muted);">↑ ' + pct + '%</span>';
|
||||
renderTable();
|
||||
});
|
||||
contractId = resp.contract_id;
|
||||
// uploadFileChunked возвращает {contract_id} только на последнем чанке
|
||||
// но этот же resp приходит только с последнего чанка
|
||||
if (resp && resp.contract_id) {
|
||||
contractId = resp.contract_id;
|
||||
}
|
||||
// Если файл был один и он маленький (один чанк), contract_id уже есть
|
||||
var elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
|
||||
fileQueue[rowIdx].status = '<span class="status-ok">✓ ' + elapsed + 'с</span>';
|
||||
fileQueue[rowIdx].uploaded = true;
|
||||
|
||||
Reference in New Issue
Block a user