Files
contracts-flask/deploy/convert_server.py
T
2026-06-30 10:08:25 +04:00

552 lines
25 KiB
Python
Executable File

#!/usr/bin/env python3
# ⛔ НЕ МЕНЯТЬ БЕЗ РАЗРЕШЕНИЯ НАЕЛЯ ⛔
"""Contracts VM server — thin HTTP router. All logic in db/ and services/."""
from http.server import HTTPServer, BaseHTTPRequestHandler
from socketserver import ThreadingMixIn, TCPServer
from urllib.parse import urlparse, parse_qs
import json, re, os, sys
# ── Загрузка .env (если есть) ────────────────────────────────────────────
_env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), ".env")
if os.path.exists(_env_path):
with open(_env_path) as _f:
for _line in _f:
_line = _line.strip()
if _line and not _line.startswith("#") and "=" in _line:
_k, _v = _line.split("=", 1)
if _k not in os.environ:
os.environ[_k] = _v
# ── DB auto-seed ──────────────────────────────────────────────────────────
try:
from db import prompts as db_prompts
from db.connection import DB_CONFIG, execute
db_prompts.seed_defaults()
# Schema migration: classify_raw column (idempotent)
execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_raw text")
execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_input text")
execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS zip_source text")
execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS content_hash text")
# ── DB modules ────────────────────────────────────────────────────────────
from db import supplements as db_supplements
from db import documents as db_documents
from db import spec_current as db_spec_current
DB_OK = True
except Exception as _db_err:
import sys as _sys
print(f"WARNING: DB init failed ({_db_err}), running without DB", file=_sys.stderr)
DB_CONFIG = {"dbname": "NONE"}
DB_OK = False
# Stubs for DB-dependent handlers
db_prompts = None
db_supplements = None
db_documents = None
db_spec_current = None
# ── Services ──────────────────────────────────────────────────────────────
if DB_OK:
from compare.upload import handle_upload
from compare.unzip import handle_unzip
from compare.process import run_pipeline
from llm_prompt import build_prompt
else:
handle_upload = handle_unzip = run_pipeline = None
build_prompt = None
class ThreadingHTTPServer(ThreadingMixIn, HTTPServer):
daemon_threads = True
class Handler(BaseHTTPRequestHandler):
def do_OPTIONS(self):
self.send_response(200)
self._send_cors()
self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
self.send_header("Access-Control-Allow-Headers", "Content-Type")
self.end_headers()
def do_GET(self):
parsed = urlparse(self.path)
if parsed.path == "/process-v2":
self._handle_process_v2(parsed)
elif parsed.path == "/health":
self._json({"ok": True, "db": DB_CONFIG.get("dbname", "NONE")})
elif parsed.path == "/app.js" or parsed.path.startswith("/static/"):
self._handle_app_js()
elif parsed.path == "/api/supplements":
self._handle_api_supplements(parsed)
elif parsed.path == "/api/groups":
self._handle_api_groups(parsed)
elif parsed.path == "/api/batch-progress":
self._handle_api_batch_progress(parsed)
elif parsed.path == "/api/prompts":
self._handle_api_prompts_get(parsed)
elif parsed.path == "/api/prompts/list":
self._handle_api_prompts_list(parsed)
elif parsed.path.startswith("/api/documents/"):
self._handle_api_document(parsed)
else:
self.send_error(404)
def do_POST(self):
if self.path == "/upload":
self._handle_upload()
elif self.path == "/unzip-upload":
self._handle_unzip()
elif self.path == "/llm-ops":
self._handle_llm_ops()
elif self.path == "/convert-doc":
self._handle_convert_doc()
elif self.path == "/api/classify-batch":
self._handle_classify_batch()
elif self.path == "/api/apply-groups":
self._handle_apply_groups()
elif self.path == "/api/cleanup":
self._handle_cleanup()
elif self.path == "/api/prompts/save":
self._handle_api_prompts_save()
elif self.path == "/api/prompts/activate":
self._handle_api_prompts_activate()
elif self.path == "/api/prompts/delete":
self._handle_api_prompts_delete()
elif self.path == "/api/sync":
self._handle_api_sync()
else:
self.send_error(404)
def do_DELETE(self):
parsed = urlparse(self.path)
if parsed.path.startswith("/api/documents/"):
self._handle_api_document_delete(parsed)
else:
self.send_error(404)
# ── /process-v2 (SSE) ─────────────────────────────────────────────────
def _handle_process_v2(self, parsed):
params = parse_qs(parsed.query)
cid = params.get("contract_id", [None])[0]
if not cid:
self.send_error(400, "contract_id required")
return
if not re.fullmatch(r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', cid, re.I):
self.send_error(400, "invalid contract_id format")
return
self.send_response(200)
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "keep-alive")
self.end_headers()
self.wfile.write(b": ok\n\n")
self.wfile.flush()
order_ids = params.get("order", [None])[0] or ""
try:
run_pipeline(cid, order_ids, self._sse, build_prompt)
except Exception as e:
self._sse({"type": "error", "message": str(e)})
def _sse(self, data):
self.wfile.write(f"data: {json.dumps(data, ensure_ascii=False)}\n\n".encode())
self.wfile.flush()
# ── /api/supplements ──────────────────────────────────────────────────
def _handle_api_supplements(self, parsed):
params = parse_qs(parsed.query)
cid = params.get("contract_id", [None])[0]
if not cid:
self._json({"ok": False, "error": "contract_id required"}, 400)
return
rows = db_supplements.list_by_contract(cid)
self._json({"ok": True, "supplements": rows})
# ── /api/documents/<id> ──────────────────────────────────────────────
def _handle_api_document(self, parsed):
doc_id = parsed.path.split("/")[-1]
doc = db_documents.get(doc_id)
if not doc:
self._json({"ok": False, "error": "not found"}, 404)
return
self._json({
"ok": True,
"id": doc["id"],
"filename": doc["filename"],
"status": doc["status"],
"elements_json": doc.get("elements_json"),
"doc_type": doc.get("doc_type"),
"own_number": doc.get("own_number"),
"parent_number": doc.get("parent_number"),
"doc_date": doc.get("doc_date"),
"counterparty": doc.get("counterparty"),
"classify_status": doc.get("classify_status"),
"classify_raw": doc.get("classify_raw"),
"classify_input": doc.get("classify_input"),
})
def _handle_api_document_delete(self, parsed):
"""DELETE /api/documents/<id> — cascade: spec_current, spec_events, supplements, document."""
from db.connection import execute, query
doc_id = parsed.path.split("/")[-1]
supps = query("SELECT id, contract_id FROM supplements WHERE document_id = %s", (doc_id,))
for s in (supps or []):
execute(
"""DELETE FROM spec_current WHERE contract_id = %s
AND last_event_id IN (SELECT id FROM spec_events WHERE supplement_id = %s)""",
(s["contract_id"], s["id"]),
)
execute("DELETE FROM spec_events WHERE supplement_id = %s", (s["id"],))
execute("DELETE FROM supplements WHERE id = %s", (s["id"],))
execute("DELETE FROM documents WHERE id = %s", (doc_id,))
self._json({"ok": True})
# ── POST /api/sync ──────────────────────────────────────────────────
def _handle_api_sync(self):
"""Удалить ВСЕ документы, НЕ входящие в keep_ids. БД = зеркало таблицы."""
from db.connection import execute, query
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length)) if length > 0 else {}
keep_ids = set(body.get("keep_ids", []))
# Prevent DoS: too many IDs
if len(keep_ids) > 1000:
self._json({"ok": False, "error": "too many keep_ids (max 1000)"}, 400)
return
docs = query("SELECT id FROM documents", ())
deleted = 0
for d in (docs or []):
if d["id"] in keep_ids:
continue
supps = query("SELECT id, contract_id FROM supplements WHERE document_id = %s", (d["id"],))
for s in (supps or []):
execute(
"""DELETE FROM spec_current WHERE contract_id = %s
AND last_event_id IN (SELECT id FROM spec_events WHERE supplement_id = %s)""",
(s["contract_id"], s["id"]),
)
execute("DELETE FROM spec_events WHERE supplement_id = %s", (s["id"],))
execute("DELETE FROM supplements WHERE id = %s", (s["id"],))
execute("DELETE FROM documents WHERE id = %s", (d["id"],))
deleted += 1
# Удалить осиротевшие contracts (без supplements)
execute("DELETE FROM contracts WHERE id NOT IN (SELECT DISTINCT contract_id FROM supplements)")
self._json({"ok": True, "deleted": deleted})
# ── POST /api/drhider ────────────────────────────────────────────────
# ── /api/groups ?batch=X ─────────────────────────────────────────────
def _handle_api_groups(self, parsed):
params = parse_qs(parsed.query)
batch_id = params.get("batch", [None])[0]
if not batch_id:
self._json({"ok": False, "error": "batch required"}, 400)
return
from compare.grouping import group_documents
result = group_documents(batch_id)
self._json(result)
# ── /api/batch-progress ?batch=X ─────────────────────────────────────
def _handle_api_batch_progress(self, parsed):
params = parse_qs(parsed.query)
batch_id = params.get("batch", [None])[0]
if not batch_id:
self._json({"ok": False, "error": "batch required"}, 400)
return
counts = db_documents.count_by_status(batch_id)
docs = db_documents.list_by_batch(batch_id)
self._json({"ok": True, "counts": counts, "total": len(docs)})
# ── POST /classify-batch ─────────────────────────────────────────────
def _handle_classify_batch(self):
"""
POST /api/classify-batch — запустить классификацию в фоне.
ЗАЧЕМ subprocess вместо threading.Thread:
ThreadPoolExecutor(4) + HTTP-поток делят коннекты к PostgreSQL.
При 50+ файлах пул исчерпывается → сервер не отвечает → 502.
Отдельный процесс — свои коннекты, своя память.
HTTP-сервер продолжает отвечать на batch-progress.
"""
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length))
batch_id = body.get("batch_id")
if not batch_id:
self._json({"ok": False, "error": "batch_id required"}, 400)
return
from db import documents as db_docs
import subprocess, os
# Guard: если classify для этого batch уже запущен — не запускать повторно
lock_path = f"/tmp/classify_{batch_id}.lock"
if os.path.exists(lock_path):
self._json({"ok": False, "error": "classify already running for this batch"}, 409)
return
# Посчитать сколько файлов — ответить сразу, classify в отдельном процессе
pending = db_docs.list_pending(batch_id)
total = len(pending)
if total == 0:
self._json({"ok": False, "error": "no pending documents"}, 400)
return
# Запустить classify_worker.py в отдельном процессе
# Лог в /home/naeel/contracts/logs/ вместо DEVNULL
log_dir = "/home/naeel/nubes/contracts/logs"
os.makedirs(log_dir, exist_ok=True)
log_path = os.path.join(log_dir, f"classify_{batch_id}.log")
log_file = open(log_path, "w")
# Создать lock-файл (воркер удалит при завершении)
with open(lock_path, "w") as lf:
lf.write(str(os.getpid()))
worker_path = os.path.join(os.path.dirname(__file__), "classify_worker.py")
subprocess.Popen(
[sys.executable, worker_path, batch_id],
stdout=log_file, stderr=subprocess.STDOUT,
)
self._json({"ok": True, "total": total, "log": log_path}, 202)
# ── POST /apply-groups ───────────────────────────────────────────────
def _handle_apply_groups(self):
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length))
batch_id = body.get("batch_id")
groups = body.get("groups", [])
if not batch_id:
self._json({"ok": False, "error": "batch_id required"}, 400)
return
from compare.grouping import apply_groups
result = apply_groups(batch_id, groups)
self._json(result)
# ── POST /api/cleanup ────────────────────────────────────────────────
def _handle_cleanup(self):
"""Полная очистка всех данных (кроме prompts). Вызывается при загрузке страницы."""
from db.connection import execute
execute("DELETE FROM spec_current")
execute("DELETE FROM spec_events")
execute("DELETE FROM supplements")
execute("DELETE FROM upload_chunks")
execute("DELETE FROM documents")
execute("DELETE FROM contracts")
self._json({"ok": True, "message": "all data cleaned"})
# ── GET /api/cleanup ───────────────────────────────────────────────────
def _handle_cleanup(self):
"""Полная очистка всех данных (кроме prompts)."""
from db.connection import execute
execute("DELETE FROM spec_current")
execute("DELETE FROM spec_events")
execute("DELETE FROM supplements")
execute("DELETE FROM upload_chunks")
execute("DELETE FROM documents")
execute("DELETE FROM contracts")
self._json({"ok": True, "message": "all data cleaned"})
# ── Prompts CRUD ──────────────────────────────────────────────────────
def _handle_api_prompts_get(self, parsed):
from db import prompts as db_p
params = parse_qs(parsed.query)
role = params.get("role", [None])[0]
if not role:
self._json({"ok": False, "error": "role required"}, 400)
return
p = db_p.get_active(role)
if p:
self._json({"ok": True, "id": p["id"], "role": p["role"], "name": p["name"],
"body": p["body"], "is_active": p["is_active"], "notes": p.get("notes"),
"created_at": str(p.get("created_at", ""))})
else:
self._json({"ok": False, "error": "not found"}, 404)
def _handle_api_prompts_list(self, parsed):
from db import prompts as db_p
params = parse_qs(parsed.query)
role = params.get("role", [None])[0]
if not role:
self._json({"ok": False, "error": "role required"}, 400)
return
versions = db_p.list_by_role(role)
self._json({"ok": True, "versions": versions})
def _handle_api_prompts_save(self):
from db import prompts as db_p
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length))
role = body.get("role", "")
name = body.get("name", "v" + __import__("datetime").datetime.now().isoformat()[:16])
prompt_body = body.get("body", "")
notes = body.get("notes", "")
is_active = body.get("is_active", True)
if not role or not prompt_body:
self._json({"ok": False, "error": "role and body required"}, 400)
return
p = db_p.save_new_version(role, name, prompt_body, notes, is_active)
self._json({"ok": True, "id": p["id"]})
def _handle_api_prompts_activate(self):
from db import prompts as db_p
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length))
pid = body.get("id", "")
if not pid:
self._json({"ok": False, "error": "id required"}, 400)
return
ok = db_p.activate(pid)
self._json({"ok": ok})
def _handle_api_prompts_delete(self):
from db import prompts as db_p
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length))
pid = body.get("id", "")
if not pid:
self._json({"ok": False, "error": "id required"}, 400)
return
ok = db_p.delete_prompt(pid)
self._json({"ok": ok})
# ── /app.js (static) ───────────────────────────────────────────────────
def _handle_app_js(self):
try:
base = os.path.realpath(os.path.dirname(__file__))
rel = urlparse(self.path).path.lstrip("/")
if rel.startswith("static/"):
rel = rel[len("static/"):]
# only .js and .svg
if not (rel.endswith(".js") or rel.endswith(".svg")):
self.send_error(404)
return
target = os.path.realpath(os.path.join(base, rel))
if not target.startswith(base + os.sep):
self.send_error(403)
return
with open(target, "rb") as f:
js = f.read()
self.send_response(200)
self.send_header("Content-Type", "application/javascript; charset=utf-8")
self.send_header("Content-Length", str(len(js)))
self.send_header("X-Content-Type-Options", "nosniff")
self._send_cors()
self.end_headers()
self.wfile.write(js)
except FileNotFoundError:
self.send_error(404)
# ── /upload ───────────────────────────────────────────────────────────
def _handle_upload(self):
try:
content_type = self.headers.get("Content-Type", "")
content_length = int(self.headers.get("Content-Length", 0))
result = handle_upload(self.rfile, content_type, content_length)
self._json(result)
except Exception as e:
self._json({"ok": False, "error": str(e)}, 500)
# ── /unzip-upload ─────────────────────────────────────────────────────
def _handle_unzip(self):
try:
content_length = int(self.headers.get("Content-Length", 0))
result = handle_unzip(self.rfile, content_length)
self._json(result)
except Exception as e:
self._json({"ok": False, "error": str(e)}, 500)
# ── /convert-doc ──────────────────────────────────────────────────────
def _handle_convert_doc(self):
"""DOC → DOCX через LibreOffice."""
import subprocess, tempfile
length = int(self.headers.get("Content-Length", 0))
data = self.rfile.read(length)
with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as f:
f.write(data)
doc_path = f.name
tmpdir = tempfile.mkdtemp()
try:
subprocess.run(
["libreoffice", "--headless", "--convert-to", "docx", "--outdir", tmpdir, doc_path],
timeout=30, capture_output=True,
)
docx_files = [x for x in os.listdir(tmpdir) if x.endswith(".docx")]
if docx_files:
with open(os.path.join(tmpdir, docx_files[0]), "rb") as f:
result = f.read()
self.send_response(200)
self.send_header("Content-Type", "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
self.send_header("Content-Length", str(len(result)))
self._send_cors()
self.end_headers()
self.wfile.write(result)
else:
self.send_error(500, "conversion produced no output")
except Exception as e:
self.send_error(500, str(e))
finally:
os.unlink(doc_path)
for x in os.listdir(tmpdir):
os.unlink(os.path.join(tmpdir, x))
os.rmdir(tmpdir)
# ── /llm-ops (debug) ──────────────────────────────────────────────────
def _handle_llm_ops(self):
from compare.llm import call_llm
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length))
try:
result, prompt_id = call_llm(
body.get("current_spec", []),
body.get("doc_text", ""),
build_prompt,
)
self._json(result)
except Exception as e:
self._json({"error": str(e)}, 502)
# ── Helpers ───────────────────────────────────────────────────────────
def _json(self, data, status=200):
self.send_response(status)
self._send_cors()
self.send_header("Content-Type", "application/json; charset=utf-8")
self.end_headers()
self.wfile.write(json.dumps(data, ensure_ascii=False).encode())
def _send_cors(self):
self.send_header("Access-Control-Allow-Origin", "*")
def log_message(self, format, *args):
pass
if __name__ == "__main__":
import os
TCPServer.allow_reuse_address = True
port = int(os.environ.get("PORT", "8766"))
server = ThreadingHTTPServer(("0.0.0.0", port), Handler)
db_name = DB_CONFIG.get("dbname", "NONE") if DB_CONFIG else "NONE"
print(f"Contracts VM server on :{port}, db={db_name}")
try:
server.serve_forever()
except KeyboardInterrupt:
server.shutdown()