v3.0.0 VM: убрать v2/Redis/чанки, чистый FormData, PostgreSQL
This commit is contained in:
+1
-12
@@ -4,9 +4,6 @@ from dotenv import load_dotenv
|
||||
from flask import Flask, render_template, request, redirect, url_for, Response, jsonify
|
||||
import json
|
||||
|
||||
import sys as _sys, os as _os
|
||||
_sys.path.insert(0, _os.path.join(_os.path.dirname(__file__), '..'))
|
||||
|
||||
import schema
|
||||
import db
|
||||
import parser as parser_mod
|
||||
@@ -17,9 +14,6 @@ import mimeutil
|
||||
from test_routes import test_bp
|
||||
from upload import upload_bp
|
||||
from api import api_bp
|
||||
from v2.chunk_api import bp as v2_bp
|
||||
import redis_client
|
||||
import redis_worker
|
||||
|
||||
load_dotenv()
|
||||
|
||||
@@ -28,8 +22,6 @@ import re as _re
|
||||
|
||||
# Логи в памяти (быстро, не тормозит ответ)
|
||||
_log_memory = []
|
||||
# Хранилище чанков в памяти
|
||||
_chunks_mem = {}
|
||||
|
||||
def _log(step, detail=""):
|
||||
"""Писать лог в память (мгновенно)."""
|
||||
@@ -83,20 +75,17 @@ class ContractsApp:
|
||||
def __init__(self):
|
||||
self.app = Flask(__name__)
|
||||
schema.ensure_schema()
|
||||
redis_worker.start_worker()
|
||||
self._add_routes()
|
||||
|
||||
def _add_routes(self):
|
||||
self.app.add_url_rule("/", "index", self._index, methods=["GET", "POST"])
|
||||
self.app.add_url_rule("/parse/<cid>", "parse", self._parse, methods=["GET"])
|
||||
self.app.add_url_rule("/chunk_json", "chunk_json", self._chunk_json, methods=["POST"])
|
||||
self.app.add_url_rule("/upload", "upload_json", self._upload_json, methods=["POST"])
|
||||
|
||||
self.app.add_url_rule("/health", "health", self._health)
|
||||
self.app.add_url_rule("/logs", "logs", self._logs)
|
||||
self.app.register_blueprint(test_bp)
|
||||
self.app.register_blueprint(upload_bp)
|
||||
self.app.register_blueprint(api_bp)
|
||||
self.app.register_blueprint(v2_bp)
|
||||
|
||||
def _health(self):
|
||||
return "OK", 200, {"Content-Type": "text/plain"}
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
"""redis_client.py — Очередь загрузок через Redis."""
|
||||
|
||||
import os, json, redis
|
||||
|
||||
UPLOAD_QUEUE = "contracts:upload_queue"
|
||||
|
||||
_r = None
|
||||
|
||||
def _get():
|
||||
global _r
|
||||
if _r is None:
|
||||
_r = redis.Redis(
|
||||
host=os.getenv("REDIS_HOST", "redisk8s.f6303a9d-4ab1-4b2f-8aa8-08f933d02f82.svc.cluster.local"),
|
||||
port=int(os.getenv("REDIS_PORT", "6379")),
|
||||
password=os.getenv("REDIS_PASS", "aLITloRefJEiCPqUc2xB"),
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=5,
|
||||
socket_timeout=10,
|
||||
)
|
||||
return _r
|
||||
|
||||
|
||||
def push(task: dict):
|
||||
"""Добавить задание в очередь."""
|
||||
_get().rpush(UPLOAD_QUEUE, json.dumps(task, ensure_ascii=False))
|
||||
|
||||
|
||||
def pop(timeout=5):
|
||||
"""Взять задание из очереди (блокирующий)."""
|
||||
result = _get().blpop(UPLOAD_QUEUE, timeout=timeout)
|
||||
if result:
|
||||
return json.loads(result[1])
|
||||
return None
|
||||
@@ -1,69 +0,0 @@
|
||||
"""redis_worker.py — Фоновый обработчик очереди загрузок."""
|
||||
|
||||
import threading, time, base64, json
|
||||
import db, mimeutil, parser as parser_mod, textify
|
||||
|
||||
def start_worker():
|
||||
"""Запустить фоновый поток обработки очереди."""
|
||||
t = threading.Thread(target=_worker_loop, daemon=True)
|
||||
t.start()
|
||||
|
||||
def _worker_loop():
|
||||
from redis_client import pop as queue_pop
|
||||
while True:
|
||||
try:
|
||||
task = queue_pop(timeout=5)
|
||||
if task:
|
||||
_process(task)
|
||||
except Exception as e:
|
||||
time.sleep(1)
|
||||
|
||||
def _process(task):
|
||||
filename = task.get("filename", "")
|
||||
b64 = task.get("b64", "")
|
||||
mime = task.get("mime", "application/octet-stream")
|
||||
cid = task.get("cid", "")
|
||||
|
||||
if not filename or not b64:
|
||||
return
|
||||
|
||||
# Декодировать и сохранить в БД
|
||||
file_bytes = base64.b64decode(b64)
|
||||
|
||||
conn, err = db.connect()
|
||||
if err:
|
||||
return
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"INSERT INTO documents (filename, mime_type, original_bytes, original_b64, status) VALUES (%s,%s,%s,%s,'uploaded')",
|
||||
(filename, mime, file_bytes, b64),
|
||||
)
|
||||
cur.execute("SELECT id FROM documents WHERE filename=%s AND status='uploaded' ORDER BY created_at DESC LIMIT 1", (filename,))
|
||||
row = cur.fetchone()
|
||||
doc_id = row[0] if row else None
|
||||
if doc_id:
|
||||
cur.execute(
|
||||
"INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')",
|
||||
(cid, str(doc_id)),
|
||||
)
|
||||
|
||||
# Парсинг
|
||||
pr = parser_mod.parse(file_bytes, mime)
|
||||
elements = pr.get("elements", [])
|
||||
if mime == "application/zip" and "files" in pr:
|
||||
for zf in pr["files"]:
|
||||
elements.extend(zf.get("elements", []))
|
||||
text = textify.to_text(elements) if elements else ""
|
||||
|
||||
cur.execute(
|
||||
"UPDATE documents SET parsed_text=%s, elements_json=%s, status='parsed' WHERE id=%s",
|
||||
(text, json.dumps(elements, ensure_ascii=False), str(doc_id)),
|
||||
)
|
||||
conn.commit()
|
||||
cur.close()
|
||||
except Exception as e:
|
||||
try: conn.rollback()
|
||||
except: pass
|
||||
finally:
|
||||
db.put_conn(conn)
|
||||
+25
-79
@@ -5,8 +5,6 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Сверка договоров</title>
|
||||
<link rel="icon" type="image/png" href="{{ url_for('static', filename='favicon.png') }}">
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/spark-md5/3.0.2/spark-md5.min.js"></script>
|
||||
<script src="https://unpkg.com/lucide@latest"></script>
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js"></script>
|
||||
<style>
|
||||
:root {
|
||||
@@ -60,7 +58,7 @@
|
||||
<body>
|
||||
<div class="topbar">
|
||||
<img src="{{ url_for('static', filename='nubes-logo.svg') }}" alt="Nubes">
|
||||
<span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v2.0.8</span></span>
|
||||
<span class="title">Сверка договоров <span style="font-weight:400;color:var(--muted);font-size:12px;">v3.0.0</span></span>
|
||||
</div>
|
||||
|
||||
<div class="content">
|
||||
@@ -156,81 +154,35 @@
|
||||
|
||||
window.removeFile = removeFile;
|
||||
|
||||
// ── Чанковая загрузка v2 (30KB + MD5) ────────────────────
|
||||
// ── Простая загрузка FormData (ВМ, без лимитов) ──────────
|
||||
|
||||
function uploadFileChunked(file, cid, onProgress) {
|
||||
function uploadFile(file, onProgress) {
|
||||
return new Promise(function(resolve, reject) {
|
||||
var reader = new FileReader();
|
||||
reader.onload = function() {
|
||||
var buffer = reader.result;
|
||||
var bytes = new Uint8Array(buffer);
|
||||
// Чанковая конвертация в base64 — apply не тянет >125K аргументов
|
||||
var b64Chunks = [];
|
||||
var B64_CHUNK = 0x8000; // 32768 bytes
|
||||
for (var bi = 0; bi < bytes.length; bi += B64_CHUNK) {
|
||||
b64Chunks.push(String.fromCharCode.apply(null, bytes.subarray(bi, bi + B64_CHUNK)));
|
||||
}
|
||||
var b64 = btoa(b64Chunks.join(''));
|
||||
var checksum = SparkMD5.ArrayBuffer.hash(buffer);
|
||||
var CHUNK = 60 * 1024; // ближе к лимиту 64KB → меньше чанков
|
||||
var totalChunks = Math.ceil(b64.length / CHUNK);
|
||||
var uploadId = 'u' + Date.now().toString(36) + Math.random().toString(36).substr(2, 7);
|
||||
var done = 0;
|
||||
var fd = new FormData();
|
||||
fd.append('files', file);
|
||||
if (contractId) fd.append('cid', contractId);
|
||||
|
||||
console.log('uploadFileChunked: ' + file.name + ' b64=' + b64.length + ' chunks=' + totalChunks + ' uploadId=' + uploadId);
|
||||
|
||||
function fetchWithRetry(url, body, retries) {
|
||||
retries = retries || 3;
|
||||
return fetch(url, {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify(body)
|
||||
}).catch(function(e) {
|
||||
if (retries > 0) {
|
||||
console.log('retry ' + url + ' (' + retries + ' left): ' + e.message);
|
||||
return new Promise(function(r) { setTimeout(r, 1000); }).then(function() {
|
||||
return fetchWithRetry(url, body, retries - 1);
|
||||
});
|
||||
}
|
||||
throw e;
|
||||
});
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open('POST', '/');
|
||||
xhr.upload.onprogress = function(e) {
|
||||
if (e.lengthComputable && onProgress) {
|
||||
onProgress(Math.round(e.loaded / e.total * 100));
|
||||
}
|
||||
|
||||
function sendChunk(i) {
|
||||
if (i >= totalChunks) {
|
||||
console.log('finalize: ' + uploadId);
|
||||
fetchWithRetry('/v2/finalize', {upload_id: uploadId, filename: file.name, total: totalChunks, checksum: checksum})
|
||||
.then(function(r) {
|
||||
if (!r.ok) throw new Error('HTTP ' + r.status);
|
||||
return r.json();
|
||||
}).then(function(data) {
|
||||
if (data.ok) resolve({contract_id: data.contract_id});
|
||||
else reject(new Error(data.error));
|
||||
}).catch(function(e) { reject(e); });
|
||||
return;
|
||||
}
|
||||
var piece = b64.substring(i * CHUNK, (i + 1) * CHUNK);
|
||||
console.log('chunk ' + i + '/' + totalChunks + ' send ' + piece.length + ' bytes');
|
||||
fetchWithRetry('/v2/chunk', {upload_id: uploadId, index: i, total: totalChunks, data: piece})
|
||||
.then(function(r) {
|
||||
if (!r.ok) throw new Error('HTTP ' + r.status);
|
||||
return r.json();
|
||||
}).then(function(data) {
|
||||
if (!data.ok) { reject(new Error(data.error)); return; }
|
||||
done++;
|
||||
if (onProgress) onProgress({pct: Math.round(done / totalChunks * 100), done: done, total: totalChunks});
|
||||
setTimeout(function() { sendChunk(i + 1); }, 3000);
|
||||
}).catch(function(e) { reject(e); });
|
||||
}
|
||||
sendChunk(0);
|
||||
};
|
||||
reader.onerror = function() { reject(new Error('Read error')); };
|
||||
reader.readAsArrayBuffer(file);
|
||||
xhr.onload = function() {
|
||||
try {
|
||||
var r = JSON.parse(xhr.responseText);
|
||||
if (r.contract_id) resolve(r);
|
||||
else reject(new Error(r.error || 'Неизвестная ошибка'));
|
||||
} catch(e) { reject(new Error('Некорректный ответ')); }
|
||||
};
|
||||
xhr.onerror = function() { reject(new Error('Сеть')); };
|
||||
xhr.ontimeout = function() { reject(new Error('Таймаут')); };
|
||||
xhr.timeout = 120000;
|
||||
xhr.send(fd);
|
||||
});
|
||||
}
|
||||
|
||||
// ── Старая загрузка (одним POST) ─────────────────────────────
|
||||
|
||||
// ── Выбор файла → сразу загрузка ────────────────────────────
|
||||
|
||||
fileInput.addEventListener('change', async function() {
|
||||
@@ -252,21 +204,15 @@
|
||||
var startTime = Date.now();
|
||||
|
||||
try {
|
||||
var resp = await uploadFileChunked(f, contractId, function(info) {
|
||||
fileQueue[rowIdx].status = '<span style="color:var(--muted);">↑ чанк ' + info.done + '/' + info.total + ' (' + info.pct + '%)</span>';
|
||||
var resp = await uploadFile(f, function(pct) {
|
||||
fileQueue[rowIdx].status = '<span style="color:var(--muted);">↑ ' + pct + '%</span>';
|
||||
renderTable();
|
||||
});
|
||||
// uploadFileChunked10k: прогресс по чанкам
|
||||
// но этот же resp приходит только с последнего чанка
|
||||
if (resp && resp.contract_id) {
|
||||
contractId = resp.contract_id;
|
||||
}
|
||||
// Если файл был один и он маленький (один чанк), contract_id уже есть
|
||||
if (resp && resp.contract_id) contractId = resp.contract_id;
|
||||
var elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
|
||||
fileQueue[rowIdx].status = '<span class="status-ok">✓ ' + elapsed + 'с</span>';
|
||||
fileQueue[rowIdx].uploaded = true;
|
||||
renderTable();
|
||||
|
||||
if (f.type === 'application/zip' || f.name.toLowerCase().endsWith('.zip')) {
|
||||
await showZipContents(f);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user