убрал LLM: только парсинг с интерактивным прогрессом в таблице

This commit is contained in:
2026-06-16 14:49:52 +04:00
parent 84ce99b189
commit 3cffca73b3
2 changed files with 224 additions and 180 deletions
+34 -65
View File
@@ -1,7 +1,7 @@
"""app.py — Точка входа и сборка слоёв."""
from dotenv import load_dotenv
from flask import Flask, render_template, request, redirect, url_for, Response
from flask import Flask, render_template, request, redirect, url_for, Response, jsonify
import json
import schema
@@ -26,7 +26,7 @@ class ContractsApp:
def _add_routes(self):
self.app.add_url_rule("/", "index", self._index, methods=["GET", "POST"])
self.app.add_url_rule("/process/<cid>", "process", self._process, methods=["GET"])
self.app.add_url_rule("/parse/<cid>", "parse", self._parse, methods=["GET"])
self.app.add_url_rule("/health", "health", self._health)
self.app.register_blueprint(test_bp)
self.app.register_blueprint(upload_bp)
@@ -80,27 +80,13 @@ class ContractsApp:
(filename,),
)
doc_id = doc["id"] if doc else None
if not doc_id:
continue
# Парсинг (быстро, без LLM)
pr = parser_mod.parse(file_bytes, mime)
elements = []
if mime == "application/zip" and "files" in pr:
for zf in pr["files"]:
elements.extend(zf.get("elements", []))
else:
elements = pr.get("elements", [])
text = textify.to_text(elements) if elements else ""
db.execute("UPDATE documents SET parsed_text=%s, status='parsed' WHERE id=%s", (text, str(doc_id)))
if doc_id:
db.execute(
"INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')",
(str(contract_id), str(doc_id)),
)
return redirect("/?id=" + str(contract_id))
return jsonify({"contract_id": str(contract_id)})
def _show_contract(self, cid):
"""Показать договор с кнопкой «Обработать»."""
@@ -132,38 +118,35 @@ class ContractsApp:
contract=contract, supplements=supp_list,
all_rows=all_rows, unprocessed=unprocessed)
# ── Шаг 2: LLM-обработка (SSE) ──────────────────────────────
# ── Шаг 2: Парсинг (SSE) ────────────────────────────────────
def _process(self, cid):
"""SSE-поток: обработка допников с интерактивным прогрессом."""
def _parse(self, cid):
"""SSE-поток: парсинг файлов договора (без LLM)."""
import time as time_mod
def generate():
start_time = time_mod.time()
total_bytes = 0
files_processed = 0
total_rows = 0
supps, _ = db.query(
"SELECT s.id, d.id as doc_id, d.parsed_text, d.filename, "
"SELECT s.id, d.id as doc_id, d.filename, d.mime_type, "
"LENGTH(d.original_bytes) as file_size "
"FROM supplements s JOIN documents d ON s.document_id=d.id "
"WHERE s.contract_id=%s AND d.parsed_text IS NOT NULL",
"WHERE s.contract_id=%s",
(cid,),
)
if not supps:
yield f"data: {json.dumps({'type': 'error', 'message': 'Нет файлов для обработки'})}\n\n"
yield f"data: {json.dumps({'type': 'error', 'message': 'Нет файлов'})}\n\n"
return
supp_rows = [dict(zip(supps["columns"], r)) for r in supps["rows"]]
for s in supp_rows:
text = s["parsed_text"]
if not text:
continue
# Проверим, не обработан ли уже
existing, _ = db.query("SELECT 1 FROM spec_rows WHERE supplement_id=%s LIMIT 1", (s["id"],))
# Пропустить уже распарсенные
existing, _ = db.query(
"SELECT 1 FROM documents WHERE id=%s AND status='parsed'", (s["doc_id"],)
)
if existing and existing["rows"]:
continue
@@ -173,51 +156,37 @@ class ContractsApp:
yield f"data: {json.dumps({'type': 'file_start', 'name': s['filename'], 'bytes': file_bytes})}\n\n"
ext = extractor.extract(text)
if "error" in ext:
db.execute("UPDATE documents SET status='extract_error', error_message=%s WHERE id=%s",
(ext["error"], s["doc_id"]))
yield f"data: {json.dumps({'type': 'file_error', 'name': s['filename'], 'error': ext['error']})}\n\n"
# Получить байты
doc, _ = db.query_one("SELECT original_bytes FROM documents WHERE id=%s", (s["doc_id"],))
if not doc or not doc.get("original_bytes"):
yield f"data: {json.dumps({'type': 'file_error', 'name': s['filename'], 'error': 'Нет данных'})}\n\n"
continue
rows = ext.get("rows", [])
total_rows += len(rows)
raw = doc["original_bytes"]
file_data = bytes(raw) if isinstance(raw, memoryview) else raw
for row in rows:
# Парсинг
pr = parser_mod.parse(file_data, s["mime_type"])
elements = []
if s["mime_type"] == "application/zip" and "files" in pr:
for zf in pr["files"]:
elements.extend(zf.get("elements", []))
else:
elements = pr.get("elements", [])
text = textify.to_text(elements) if elements else ""
db.execute(
"INSERT INTO spec_rows (supplement_id,row_num,name,price,qty,sum,date_start) VALUES (%s,%s,%s,%s,%s,%s,%s)",
(s["id"], row.get("row_num"), row.get("name"),
row.get("price"), row.get("qty"), row.get("sum"), row.get("date_start")),
"UPDATE documents SET parsed_text=%s, status='parsed' WHERE id=%s",
(text, str(s["doc_id"])),
)
# Diff
prev_res, _ = db.query(
"SELECT sr.row_num,sr.name,sr.price,sr.qty,sr.sum,sr.date_start "
"FROM spec_rows sr JOIN supplements sup ON sr.supplement_id=sup.id "
"WHERE sup.contract_id=%s AND sup.id!=%s ORDER BY sr.row_num",
(cid, s["id"]),
)
prev_rows = [dict(zip(prev_res["columns"], r)) for r in prev_res["rows"]] if prev_res else []
diff_r = differ.diff(prev_rows, rows)
for ch in diff_r.get("changes", []):
db.execute(
"INSERT INTO spec_history (contract_id,supplement_id,row_num,change_type,old_values,new_values) "
"VALUES (%s,%s,%s,%s,%s,%s)",
(cid, s["id"], ch["row_num"], ch["change_type"],
json.dumps(ch.get("old_values"), ensure_ascii=False, default=str) if ch.get("old_values") else None,
json.dumps(ch.get("new_values"), ensure_ascii=False, default=str) if ch.get("new_values") else None),
)
db.execute("UPDATE documents SET status='extracted' WHERE id=%s", (s["doc_id"],))
elapsed = round(time_mod.time() - file_start, 1)
files_processed += 1
yield f"data: {json.dumps({'type': 'file_done', 'name': s['filename'], 'rows': len(rows), 'time_s': elapsed})}\n\n"
yield f"data: {json.dumps({'type': 'file_done', 'name': s['filename'], 'time_s': elapsed, 'elements': len(elements)})}\n\n"
total_time = round(time_mod.time() - start_time, 1)
yield f"data: {json.dumps({'type': 'summary', 'total_time_s': total_time, 'total_bytes': total_bytes, 'total_rows': total_rows, 'files_processed': files_processed})}\n\n"
yield f"data: {json.dumps({'type': 'summary', 'total_time_s': total_time, 'total_bytes': total_bytes, 'files_processed': files_processed})}\n\n"
return Response(generate(), mimetype="text/event-stream")
+185 -110
View File
@@ -6,6 +6,7 @@
<title>Сверка договоров</title>
<link rel="icon" type="image/png" href="{{ url_for('static', filename='favicon.png') }}">
<script src="https://unpkg.com/lucide@latest"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js"></script>
<style>
:root {
--brand-primary: #2563eb; --brand-primary-dark: #1d4ed8;
@@ -18,24 +19,28 @@
.topbar { background: var(--background); border-bottom: 1px solid var(--brand-gray); padding: 0 24px; height: 56px; display: flex; align-items: center; gap: 16px; }
.topbar img { height: 28px; }
.topbar .title { font-weight: 600; font-size: 16px; }
.content { max-width: 800px; margin: 32px auto; padding: 0 16px; }
.content { max-width: 900px; margin: 32px auto; padding: 0 16px; }
.card { background: var(--background); border-radius: 12px; border: 1px solid var(--brand-gray); box-shadow: 0 1px 2px rgba(0,0,0,.05); overflow: hidden; margin-bottom: 16px; }
.card-header { background: var(--brand-grey-light); padding: 12px 16px; font-weight: 600; font-size: 16px; display: flex; align-items: center; gap: 8px; }
.card-body { padding: 16px; }
.btn { height: 36px; border-radius: 6px; gap: 6px; padding: 0 14px; font-size: 14px; font-family: inherit; cursor: pointer; display: inline-flex; align-items: center; border: 1px solid var(--brand-gray); background: var(--background); }
.btn-primary { background: var(--brand-primary); border-color: var(--brand-primary); color: #fff; }
.btn-primary:hover { background: var(--brand-primary-dark); }
.queue-item { display: flex; align-items: center; gap: 8px; padding: 8px 12px; border-radius: 6px; border: 1px solid var(--brand-gray); background: var(--brand-grey-light); margin-bottom: 4px; }
.queue-item .name { flex: 1; font-weight: 500; }
.queue-item .size { color: var(--muted); font-size: 12px; }
.btn:disabled { opacity: .5; cursor: not-allowed; }
.table-wrap { border-radius: 6px; border: 1px solid var(--brand-gray); overflow: hidden; margin-top: 12px; }
table { width: 100%; border-collapse: collapse; font-size: 13px; }
th { background: var(--brand-grey-light); text-transform: uppercase; padding: 4px 8px; border-right: 1px solid var(--brand-gray); text-align: left; font-weight: 600; }
td { padding: 4px 8px; border-right: 1px solid var(--brand-gray); border-bottom: 1px solid var(--brand-gray); }
th { background: var(--brand-grey-light); text-transform: uppercase; padding: 6px 10px; border-right: 1px solid var(--brand-gray); text-align: left; font-weight: 600; font-size: 11px; color: var(--muted); }
td { padding: 6px 10px; border-right: 1px solid var(--brand-gray); border-bottom: 1px solid var(--brand-gray); }
tr:hover td { background: rgba(243,244,246,.5); }
.badge { display: inline-flex; align-items: center; gap: 4px; font-size: 12px; padding: 2px 8px; border-radius: 4px; }
.badge-ok { background: rgba(34,197,94,.1); color: var(--green); }
.badge-err { background: rgba(239,68,68,.1); color: var(--destructive); }
.name-cell { max-width: 380px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.zip-child .name-cell { padding-left: 28px; }
.zip-child .name-cell::before { content: "└ "; color: var(--muted); }
.status-ok { color: var(--green); }
.status-err { color: var(--destructive); }
.summary-row td { background: rgba(34,197,94,.05); font-weight: 600; }
.empty-row td { color: var(--muted); text-align: center; padding: 24px; }
.remove-btn { cursor: pointer; color: var(--muted); background: none; border: none; padding: 2px 4px; font-size: 16px; line-height: 1; }
.remove-btn:hover { color: var(--destructive); }
.error-box { margin-top: 12px; padding: 12px; border-radius: 6px; border: 1px solid var(--destructive); background: rgba(239,68,68,.05); color: var(--destructive); }
</style>
</head>
@@ -52,149 +57,219 @@
Загрузка договоров / допников
</div>
<div class="card-body">
<form method="POST" enctype="multipart/form-data">
<input type="file" name="files" accept=".docx,.doc,.pdf,.zip" multiple style="margin-bottom:12px;width:100%;">
<button type="submit" class="btn btn-primary" style="width:100%;justify-content:center;">
<i data-lucide="upload" style="width:16px;height:16px;"></i> Загрузить
</button>
</form>
</div>
</div>
<input type="file" id="fileInput" accept=".docx,.doc,.pdf,.zip" multiple style="margin-bottom:12px;width:100%;">
{% if contract %}
<div class="card">
<div class="card-header">
<i data-lucide="file-check" style="width:18px;height:18px;"></i>
Договор {{ contract.number }} — файлы загружены
</div>
<div class="card-body">
{% for s in supplements %}
<div class="queue-item">
<span class="badge {{ 'badge-ok' if s.status == 'extracted' else 'badge-err' }}">{{ '✅' if s.status == 'extracted' else '⏳' }}</span>
<span class="name">{{ s.filename }}</span>
<span class="size">{{ s.status }}</span>
</div>
{% endfor %}
{% if unprocessed %}
<div id="process-area">
<button class="btn btn-primary" onclick="startProcessing('{{ contract.id }}')" style="width:100%;justify-content:center;">
<i data-lucide="play" style="width:16px;height:16px;"></i> Обработать (LLM)
</button>
</div>
{% endif %}
</div>
</div>
{% endif %}
{% if all_rows %}
<div class="card">
<div class="card-header">
<i data-lucide="file-text" style="width:18px;height:18px;"></i>
Результаты — строк: {{ all_rows|length }}
</div>
<div class="card-body">
<div class="table-wrap">
<table>
<thead><tr><th></th><th>Услуга</th><th>Цена, руб</th><th>Кол-во</th><th>Сумма, руб</th><th>Дата</th></tr></thead>
<tbody>
{% for row in all_rows %}
<tr>
<td>{{ row.row_num }}</td>
<td>{{ row.name[:80] if row.name else '' }}</td>
<td style="text-align:right">{{ '{:,.2f}'.format(row.price) if row.price else '' }}</td>
<td style="text-align:center">{{ row.qty if row.qty else '' }}</td>
<td style="text-align:right">{{ '{:,.2f}'.format(row.sum) if row.sum else '' }}</td>
<td>{{ row.date_start if row.date_start else '' }}</td>
</tr>
{% endfor %}
</tbody>
<thead>
<tr><th>Имя</th><th style="width:130px;">Изменён</th><th style="width:80px;">Размер</th><th style="width:100px;">Статус</th><th style="width:30px;"></th></tr>
</thead>
<tbody id="fileTable"></tbody>
</table>
</div>
<button class="btn btn-primary" id="parseBtn" style="width:100%;justify-content:center;margin-top:12px;" disabled>
<i data-lucide="play" style="width:16px;height:16px;"></i> Парсинг
</button>
</div>
</div>
{% endif %}
</div>
<script>
lucide.createIcons();
function formatBytes(b) {
if (!b) return '0 B';
var u = ['B','KB','MB','GB'], i = 0;
while (b >= 1024 && i < 3) { b /= 1024; i++; }
return (Math.round(b * 10) / 10) + ' ' + u[i];
const fileInput = document.getElementById('fileInput');
const parseBtn = document.getElementById('parseBtn');
const fileTable = document.getElementById('fileTable');
let fileQueue = [];
function formatSize(bytes) {
if (bytes < 1024) return bytes + ' B';
if (bytes < 1048576) return (bytes / 1024).toFixed(1) + ' KB';
return (bytes / 1048576).toFixed(1) + ' MB';
}
function startProcessing(cid) {
var area = document.getElementById('process-area');
area.innerHTML = '<div style="padding:12px;" id="proc-list"></div>';
var list = document.getElementById('proc-list');
function formatDate(ts) {
if (!ts) return '—';
var d = new Date(ts);
return d.toLocaleDateString('ru-RU') + ' ' + d.toLocaleTimeString('ru-RU', {hour:'2-digit',minute:'2-digit'});
}
var timers = {};
function renderTable() {
if (fileQueue.length === 0) {
fileTable.innerHTML = '<tr class="empty-row"><td colspan="5">Нет файлов — выберите .docx / .pdf / .zip</td></tr>';
parseBtn.disabled = true;
} else {
parseBtn.disabled = false;
fileTable.innerHTML = fileQueue.map(function(f, i) {
var cls = f.isZipChild ? 'zip-child' : '';
return '<tr class="' + cls + '" id="row_' + i + '">' +
'<td class="name-cell">' + f.name + '</td>' +
'<td style="font-size:12px;color:var(--muted);">' + formatDate(f.lastModified) + '</td>' +
'<td style="font-size:12px;color:var(--muted);">' + (f.size ? formatSize(f.size) : '—') + '</td>' +
'<td class="status-cell"></td>' +
'<td><button class="remove-btn" onclick="removeFile(' + i + ')" title="Удалить">×</button></td>' +
'</tr>';
}).join('');
}
}
function removeFile(index) {
var f = fileQueue[index];
if (!f) return;
// Если удаляем ZIP — удалить и всех его детей
if (!f.isZipChild && f.file && (f.file.type === 'application/zip' || f.name.toLowerCase().endsWith('.zip'))) {
for (var i = fileQueue.length - 1; i >= 0; i--) {
if (fileQueue[i].zipName === f.name) fileQueue.splice(i, 1);
}
}
fileQueue.splice(index, 1);
renderTable();
}
window.removeFile = removeFile;
fileInput.addEventListener('change', async function() {
var newFiles = Array.from(fileInput.files);
for (var i = 0; i < newFiles.length; i++) {
var f = newFiles[i];
if (fileQueue.find(function(q) { return q.name === f.name && q.size === f.size; })) continue;
fileQueue.push({
name: f.name,
lastModified: f.lastModified,
size: f.size,
file: f
});
// ZIP: показать содержимое
if (f.type === 'application/zip' || f.name.toLowerCase().endsWith('.zip')) {
try {
var zip = await JSZip.loadAsync(f);
zip.forEach(function(relativePath, zipEntry) {
if (!zipEntry.dir) {
var fname = relativePath.split('/').pop() || relativePath;
fileQueue.push({
name: fname,
lastModified: zipEntry.date ? zipEntry.date.getTime() : 0,
size: 0,
isZipChild: true,
zipName: f.name
});
}
});
} catch(e) {
console.log('JSZip error:', e);
}
}
}
fileInput.value = '';
renderTable();
});
parseBtn.addEventListener('click', async function() {
if (fileQueue.length === 0) return;
var realFiles = fileQueue.filter(function(f) { return !f.isZipChild; });
if (realFiles.length === 0) return;
parseBtn.disabled = true;
parseBtn.innerHTML = '<span style="display:inline-block;width:16px;height:16px;border:2px solid rgba(255,255,255,.3);border-top-color:#fff;border-radius:50%;animation:spin .6s linear infinite;"></span> Загрузка...';
var formData = new FormData();
realFiles.forEach(function(f) { formData.append('files', f.file); });
var contractId;
try {
var uploadResp = await fetch('/', { method: 'POST', body: formData });
var uploadJson = await uploadResp.json();
if (uploadJson.error) { alert(uploadJson.error); resetBtn(); return; }
contractId = uploadJson.contract_id;
} catch(e) {
alert('Ошибка загрузки: ' + e.message);
resetBtn();
return;
}
parseBtn.innerHTML = '<span style="display:inline-block;width:16px;height:16px;border:2px solid rgba(255,255,255,.3);border-top-color:#fff;border-radius:50%;animation:spin .6s linear infinite;"></span> Парсинг...';
var es = new EventSource('/parse/' + contractId);
var timerInterval = setInterval(function() {
var now = Date.now();
var els = list.querySelectorAll('.proc-timer');
els.forEach(function(el) {
document.querySelectorAll('.proc-timer').forEach(function(el) {
var start = parseInt(el.dataset.start);
if (start) { el.textContent = Math.round((now - start) / 1000) + 'с'; }
if (start) el.textContent = '⏳ ' + Math.round((now - start) / 1000) + 'с';
});
}, 500);
var es = new EventSource('/process/' + cid);
es.addEventListener('error', function() {
clearInterval(timerInterval);
es.close();
});
es.onmessage = function(e) {
var d = JSON.parse(e.data);
if (d.type === 'file_start') {
var div = document.createElement('div');
div.className = 'queue-item';
div.id = 'f_' + d.name.replace(/[^a-zA-Z0-9]/g, '_');
div.innerHTML = '<span class="badge badge-err">⏳</span>' +
'<span class="name">' + d.name + '</span>' +
'<span class="size proc-timer" data-start="' + Date.now() + '">0с</span>';
list.appendChild(div);
lucide.createIcons();
var row = findRowByName(d.name);
if (row) {
row.querySelector('.status-cell').innerHTML = '<span class="proc-timer" data-start="' + Date.now() + '">⏳ 0с</span>';
}
}
else if (d.type === 'file_done') {
var el = document.getElementById('f_' + d.name.replace(/[^a-zA-Z0-9]/g, '_'));
if (el) {
el.querySelector('.badge').className = 'badge badge-ok';
el.querySelector('.badge').textContent = '✓';
el.querySelector('.proc-timer').textContent = d.time_s + 'с · ' + d.rows + ' строк';
var row = findRowByName(d.name);
if (row) {
row.querySelector('.status-cell').innerHTML = '<span class="status-ok">✓ ' + d.time_s + 'с</span>';
}
}
else if (d.type === 'file_error') {
var el = document.getElementById('f_' + d.name.replace(/[^a-zA-Z0-9]/g, '_'));
if (el) {
el.querySelector('.badge').className = 'badge badge-err';
el.querySelector('.badge').textContent = '✗';
el.querySelector('.proc-timer').textContent = d.error;
var row = findRowByName(d.name);
if (row) {
row.querySelector('.status-cell').innerHTML = '<span class="status-err">✗ ' + d.error + '</span>';
}
}
else if (d.type === 'summary') {
clearInterval(timerInterval);
es.close();
var sum = document.createElement('div');
sum.style.cssText = 'margin-top:12px;padding:12px;border-radius:6px;background:rgba(34,197,94,.05);border:1px solid var(--green);';
sum.innerHTML = '<strong>✓ Готово:</strong> ' + d.files_processed + ' файлов, ' +
d.total_rows + ' строк, ' + formatBytes(d.total_bytes) + ', всего <strong>' + d.total_time_s + 'с</strong>';
list.appendChild(sum);
resetBtn();
var btnDiv = document.createElement('div');
btnDiv.style.cssText = 'margin-top:12px;';
btnDiv.innerHTML = '<a href="/?id=' + cid + '" class="btn btn-primary" style="width:100%;justify-content:center;">Смотреть результаты</a>';
list.appendChild(btnDiv);
var summaryRow = document.createElement('tr');
summaryRow.className = 'summary-row';
summaryRow.innerHTML = '<td>✓ Готово: ' + d.files_processed + ' файлов</td>' +
'<td></td>' +
'<td>' + formatSize(d.total_bytes) + '</td>' +
'<td><strong>' + d.total_time_s + 'с</strong></td>' +
'<td></td>';
fileTable.appendChild(summaryRow);
}
};
es.addEventListener('error', function() {
clearInterval(timerInterval);
es.close();
resetBtn();
});
function resetBtn() {
parseBtn.disabled = false;
parseBtn.innerHTML = '<i data-lucide="play" style="width:16px;height:16px;"></i> Парсинг';
lucide.createIcons();
}
function findRowByName(name) {
for (var i = 0; i < fileQueue.length; i++) {
if (fileQueue[i].name === name && !fileQueue[i].isZipChild) {
return document.getElementById('row_' + i);
}
}
return null;
}
});
var style = document.createElement('style');
style.textContent = '@keyframes spin { to { transform: rotate(360deg); } }';
document.head.appendChild(style);
</script>
</body>
</html>