From 12d67de7edd9ad5385fdf402e50327c8e6320031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Thu, 18 Jun 2026 07:57:51 +0400 Subject: [PATCH] =?UTF-8?q?v3.0.0=20VM:=20=D1=83=D0=B1=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D1=8C=20v2/Redis/=D1=87=D0=B0=D0=BD=D0=BA=D0=B8,=20=D1=87?= =?UTF-8?q?=D0=B8=D1=81=D1=82=D1=8B=D0=B9=20FormData,=20PostgreSQL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | 3 - site/app.py | 13 +-- site/redis_client.py | 33 ------ site/redis_worker.py | 69 ------------ site/templates/upload.html | 104 +++++------------- v2/chunk_api.py | 75 ------------- v2/chunk_upload.js | 213 ------------------------------------- v2/init.py | 45 -------- v2/worker.py | 51 --------- 9 files changed, 26 insertions(+), 580 deletions(-) delete mode 100644 site/redis_client.py delete mode 100644 site/redis_worker.py delete mode 100644 v2/chunk_api.py delete mode 100644 v2/chunk_upload.js delete mode 100644 v2/init.py delete mode 100644 v2/worker.py diff --git a/requirements.txt b/requirements.txt index db54537..e5fd237 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,3 @@ psycopg2-binary python-dotenv pdfplumber camelot-py>=2.0 -redis -httpx -h2 diff --git a/site/app.py b/site/app.py index fb53d5d..4d219b2 100644 --- a/site/app.py +++ b/site/app.py @@ -4,9 +4,6 @@ from dotenv import load_dotenv from flask import Flask, render_template, request, redirect, url_for, Response, jsonify import json -import sys as _sys, os as _os -_sys.path.insert(0, _os.path.join(_os.path.dirname(__file__), '..')) - import schema import db import parser as parser_mod @@ -17,9 +14,6 @@ import mimeutil from test_routes import test_bp from upload import upload_bp from api import api_bp -from v2.chunk_api import bp as v2_bp -import redis_client -import redis_worker load_dotenv() @@ -28,8 +22,6 @@ import re as _re # Логи в памяти (быстро, не тормозит ответ) _log_memory = [] -# Хранилище чанков в памяти -_chunks_mem = {} def _log(step, detail=""): """Писать лог в память (мгновенно).""" @@ -83,20 +75,17 @@ class ContractsApp: def __init__(self): self.app = Flask(__name__) schema.ensure_schema() - redis_worker.start_worker() self._add_routes() def _add_routes(self): self.app.add_url_rule("/", "index", self._index, methods=["GET", "POST"]) self.app.add_url_rule("/parse/", "parse", self._parse, methods=["GET"]) - self.app.add_url_rule("/chunk_json", "chunk_json", self._chunk_json, methods=["POST"]) - self.app.add_url_rule("/upload", "upload_json", self._upload_json, methods=["POST"]) + self.app.add_url_rule("/health", "health", self._health) self.app.add_url_rule("/logs", "logs", self._logs) self.app.register_blueprint(test_bp) self.app.register_blueprint(upload_bp) self.app.register_blueprint(api_bp) - self.app.register_blueprint(v2_bp) def _health(self): return "OK", 200, {"Content-Type": "text/plain"} diff --git a/site/redis_client.py b/site/redis_client.py deleted file mode 100644 index 0a59bd3..0000000 --- a/site/redis_client.py +++ /dev/null @@ -1,33 +0,0 @@ -"""redis_client.py — Очередь загрузок через Redis.""" - -import os, json, redis - -UPLOAD_QUEUE = "contracts:upload_queue" - -_r = None - -def _get(): - global _r - if _r is None: - _r = redis.Redis( - host=os.getenv("REDIS_HOST", "redisk8s.f6303a9d-4ab1-4b2f-8aa8-08f933d02f82.svc.cluster.local"), - port=int(os.getenv("REDIS_PORT", "6379")), - password=os.getenv("REDIS_PASS", "aLITloRefJEiCPqUc2xB"), - decode_responses=True, - socket_connect_timeout=5, - socket_timeout=10, - ) - return _r - - -def push(task: dict): - """Добавить задание в очередь.""" - _get().rpush(UPLOAD_QUEUE, json.dumps(task, ensure_ascii=False)) - - -def pop(timeout=5): - """Взять задание из очереди (блокирующий).""" - result = _get().blpop(UPLOAD_QUEUE, timeout=timeout) - if result: - return json.loads(result[1]) - return None diff --git a/site/redis_worker.py b/site/redis_worker.py deleted file mode 100644 index cb7c881..0000000 --- a/site/redis_worker.py +++ /dev/null @@ -1,69 +0,0 @@ -"""redis_worker.py — Фоновый обработчик очереди загрузок.""" - -import threading, time, base64, json -import db, mimeutil, parser as parser_mod, textify - -def start_worker(): - """Запустить фоновый поток обработки очереди.""" - t = threading.Thread(target=_worker_loop, daemon=True) - t.start() - -def _worker_loop(): - from redis_client import pop as queue_pop - while True: - try: - task = queue_pop(timeout=5) - if task: - _process(task) - except Exception as e: - time.sleep(1) - -def _process(task): - filename = task.get("filename", "") - b64 = task.get("b64", "") - mime = task.get("mime", "application/octet-stream") - cid = task.get("cid", "") - - if not filename or not b64: - return - - # Декодировать и сохранить в БД - file_bytes = base64.b64decode(b64) - - conn, err = db.connect() - if err: - return - try: - cur = conn.cursor() - cur.execute( - "INSERT INTO documents (filename, mime_type, original_bytes, original_b64, status) VALUES (%s,%s,%s,%s,'uploaded')", - (filename, mime, file_bytes, b64), - ) - cur.execute("SELECT id FROM documents WHERE filename=%s AND status='uploaded' ORDER BY created_at DESC LIMIT 1", (filename,)) - row = cur.fetchone() - doc_id = row[0] if row else None - if doc_id: - cur.execute( - "INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')", - (cid, str(doc_id)), - ) - - # Парсинг - pr = parser_mod.parse(file_bytes, mime) - elements = pr.get("elements", []) - if mime == "application/zip" and "files" in pr: - for zf in pr["files"]: - elements.extend(zf.get("elements", [])) - text = textify.to_text(elements) if elements else "" - - cur.execute( - "UPDATE documents SET parsed_text=%s, elements_json=%s, status='parsed' WHERE id=%s", - (text, json.dumps(elements, ensure_ascii=False), str(doc_id)), - ) - conn.commit() - cur.close() - except Exception as e: - try: conn.rollback() - except: pass - finally: - db.put_conn(conn) diff --git a/site/templates/upload.html b/site/templates/upload.html index 142d1c1..38420ac 100644 --- a/site/templates/upload.html +++ b/site/templates/upload.html @@ -5,8 +5,6 @@ Сверка договоров - -