SSE-обработка: интерактивный прогресс с таймерами, итоги по времени/байтам
This commit is contained in:
+32
-8
@@ -1,7 +1,7 @@
|
||||
"""app.py — Точка входа и сборка слоёв."""
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from flask import Flask, render_template, request, redirect, url_for
|
||||
from flask import Flask, render_template, request, redirect, url_for, Response
|
||||
import json
|
||||
|
||||
import schema
|
||||
@@ -26,7 +26,7 @@ class ContractsApp:
|
||||
|
||||
def _add_routes(self):
|
||||
self.app.add_url_rule("/", "index", self._index, methods=["GET", "POST"])
|
||||
self.app.add_url_rule("/process/<cid>", "process", self._process, methods=["POST"])
|
||||
self.app.add_url_rule("/process/<cid>", "process", self._process, methods=["GET"])
|
||||
self.app.add_url_rule("/health", "health", self._health)
|
||||
self.app.register_blueprint(test_bp)
|
||||
self.app.register_blueprint(upload_bp)
|
||||
@@ -132,21 +132,30 @@ class ContractsApp:
|
||||
contract=contract, supplements=supp_list,
|
||||
all_rows=all_rows, unprocessed=unprocessed)
|
||||
|
||||
# ── Шаг 2: LLM-обработка ────────────────────────────────────
|
||||
# ── Шаг 2: LLM-обработка (SSE) ──────────────────────────────
|
||||
|
||||
def _process(self, cid):
|
||||
"""Запустить LLM-обработку для всех необработанных допников договора."""
|
||||
"""SSE-поток: обработка допников с интерактивным прогрессом."""
|
||||
import time as time_mod
|
||||
|
||||
def generate():
|
||||
start_time = time_mod.time()
|
||||
total_bytes = 0
|
||||
files_processed = 0
|
||||
total_rows = 0
|
||||
|
||||
supps, _ = db.query(
|
||||
"SELECT s.id, d.id as doc_id, d.parsed_text, d.filename "
|
||||
"SELECT s.id, d.id as doc_id, d.parsed_text, d.filename, "
|
||||
"LENGTH(d.original_bytes) as file_size "
|
||||
"FROM supplements s JOIN documents d ON s.document_id=d.id "
|
||||
"WHERE s.contract_id=%s AND d.parsed_text IS NOT NULL",
|
||||
(cid,),
|
||||
)
|
||||
if not supps:
|
||||
return redirect("/?id=" + cid)
|
||||
yield f"data: {json.dumps({'type': 'error', 'message': 'Нет файлов для обработки'})}\n\n"
|
||||
return
|
||||
|
||||
supp_rows = [dict(zip(supps["columns"], r)) for r in supps["rows"]]
|
||||
total_rows = 0
|
||||
|
||||
for s in supp_rows:
|
||||
text = s["parsed_text"]
|
||||
@@ -158,10 +167,17 @@ class ContractsApp:
|
||||
if existing and existing["rows"]:
|
||||
continue
|
||||
|
||||
file_start = time_mod.time()
|
||||
file_bytes = s.get("file_size", 0) or 0
|
||||
total_bytes += file_bytes
|
||||
|
||||
yield f"data: {json.dumps({'type': 'file_start', 'name': s['filename'], 'bytes': file_bytes})}\n\n"
|
||||
|
||||
ext = extractor.extract(text)
|
||||
if "error" in ext:
|
||||
db.execute("UPDATE documents SET status='extract_error', error_message=%s WHERE id=%s",
|
||||
(ext["error"], s["doc_id"]))
|
||||
yield f"data: {json.dumps({'type': 'file_error', 'name': s['filename'], 'error': ext['error']})}\n\n"
|
||||
continue
|
||||
|
||||
rows = ext.get("rows", [])
|
||||
@@ -195,7 +211,15 @@ class ContractsApp:
|
||||
|
||||
db.execute("UPDATE documents SET status='extracted' WHERE id=%s", (s["doc_id"],))
|
||||
|
||||
return redirect("/?id=" + cid)
|
||||
elapsed = round(time_mod.time() - file_start, 1)
|
||||
files_processed += 1
|
||||
|
||||
yield f"data: {json.dumps({'type': 'file_done', 'name': s['filename'], 'rows': len(rows), 'time_s': elapsed})}\n\n"
|
||||
|
||||
total_time = round(time_mod.time() - start_time, 1)
|
||||
yield f"data: {json.dumps({'type': 'summary', 'total_time_s': total_time, 'total_bytes': total_bytes, 'total_rows': total_rows, 'files_processed': files_processed})}\n\n"
|
||||
|
||||
return Response(generate(), mimetype="text/event-stream")
|
||||
|
||||
def run(self):
|
||||
self.app.run(host="0.0.0.0", port=5000)
|
||||
|
||||
@@ -76,12 +76,11 @@
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% if unprocessed %}
|
||||
<form method="POST" action="/process/{{ contract.id }}" style="margin-top:12px;" onsubmit="startTimer()">
|
||||
<button type="submit" class="btn btn-primary" style="width:100%;justify-content:center;">
|
||||
<div id="process-area">
|
||||
<button class="btn btn-primary" onclick="startProcessing('{{ contract.id }}')" style="width:100%;justify-content:center;">
|
||||
<i data-lucide="play" style="width:16px;height:16px;"></i> Обработать (LLM)
|
||||
</button>
|
||||
<div id="timer" style="text-align:center;margin-top:8px;color:var(--muted);font-size:13px;"></div>
|
||||
</form>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
@@ -118,11 +117,83 @@
|
||||
|
||||
<script>
|
||||
lucide.createIcons();
|
||||
function startTimer() {
|
||||
var t = document.getElementById('timer'), s = 0;
|
||||
if (!t) return;
|
||||
t.textContent = '⏳ Обработка LLM... 0с';
|
||||
setInterval(function(){ s++; t.textContent = '⏳ Обработка LLM... ' + s + 'с'; }, 1000);
|
||||
|
||||
function formatBytes(b) {
|
||||
if (!b) return '0 B';
|
||||
var u = ['B','KB','MB','GB'], i = 0;
|
||||
while (b >= 1024 && i < 3) { b /= 1024; i++; }
|
||||
return (Math.round(b * 10) / 10) + ' ' + u[i];
|
||||
}
|
||||
|
||||
function startProcessing(cid) {
|
||||
var area = document.getElementById('process-area');
|
||||
area.innerHTML = '<div style="padding:12px;" id="proc-list"></div>';
|
||||
var list = document.getElementById('proc-list');
|
||||
|
||||
var timers = {};
|
||||
var timerInterval = setInterval(function() {
|
||||
var now = Date.now();
|
||||
var els = list.querySelectorAll('.proc-timer');
|
||||
els.forEach(function(el) {
|
||||
var start = parseInt(el.dataset.start);
|
||||
if (start) { el.textContent = Math.round((now - start) / 1000) + 'с'; }
|
||||
});
|
||||
}, 500);
|
||||
|
||||
var es = new EventSource('/process/' + cid);
|
||||
|
||||
es.addEventListener('error', function() {
|
||||
clearInterval(timerInterval);
|
||||
es.close();
|
||||
});
|
||||
|
||||
es.onmessage = function(e) {
|
||||
var d = JSON.parse(e.data);
|
||||
|
||||
if (d.type === 'file_start') {
|
||||
var div = document.createElement('div');
|
||||
div.className = 'queue-item';
|
||||
div.id = 'f_' + d.name.replace(/[^a-zA-Z0-9]/g, '_');
|
||||
div.innerHTML = '<span class="badge badge-err">⏳</span>' +
|
||||
'<span class="name">' + d.name + '</span>' +
|
||||
'<span class="size proc-timer" data-start="' + Date.now() + '">0с</span>';
|
||||
list.appendChild(div);
|
||||
lucide.createIcons();
|
||||
}
|
||||
|
||||
else if (d.type === 'file_done') {
|
||||
var el = document.getElementById('f_' + d.name.replace(/[^a-zA-Z0-9]/g, '_'));
|
||||
if (el) {
|
||||
el.querySelector('.badge').className = 'badge badge-ok';
|
||||
el.querySelector('.badge').textContent = '✓';
|
||||
el.querySelector('.proc-timer').textContent = d.time_s + 'с · ' + d.rows + ' строк';
|
||||
}
|
||||
}
|
||||
|
||||
else if (d.type === 'file_error') {
|
||||
var el = document.getElementById('f_' + d.name.replace(/[^a-zA-Z0-9]/g, '_'));
|
||||
if (el) {
|
||||
el.querySelector('.badge').className = 'badge badge-err';
|
||||
el.querySelector('.badge').textContent = '✗';
|
||||
el.querySelector('.proc-timer').textContent = d.error;
|
||||
}
|
||||
}
|
||||
|
||||
else if (d.type === 'summary') {
|
||||
clearInterval(timerInterval);
|
||||
es.close();
|
||||
var sum = document.createElement('div');
|
||||
sum.style.cssText = 'margin-top:12px;padding:12px;border-radius:6px;background:rgba(34,197,94,.05);border:1px solid var(--green);';
|
||||
sum.innerHTML = '<strong>✓ Готово:</strong> ' + d.files_processed + ' файлов, ' +
|
||||
d.total_rows + ' строк, ' + formatBytes(d.total_bytes) + ', всего <strong>' + d.total_time_s + 'с</strong>';
|
||||
list.appendChild(sum);
|
||||
|
||||
var btnDiv = document.createElement('div');
|
||||
btnDiv.style.cssText = 'margin-top:12px;';
|
||||
btnDiv.innerHTML = '<a href="/?id=' + cid + '" class="btn btn-primary" style="width:100%;justify-content:center;">Смотреть результаты</a>';
|
||||
list.appendChild(btnDiv);
|
||||
}
|
||||
};
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
|
||||
Reference in New Issue
Block a user