Steps 3-8: config, blueprints (upload/pipeline/api/prompts/health/pages), new app.py (no VM proxy), process.py adapted to generator, VM_API=''
This commit is contained in:
+53
-117
@@ -1,128 +1,64 @@
|
||||
"""contracts-flask — Flask-фронтенд для сверки договоров.
|
||||
1:1 функционал с Lucee-версией.
|
||||
Все тяжёлые операции — на ВМ (contracts.kube5s.ru).
|
||||
"""contracts-flask v2.0 — полный перенос с ВМ на Flask.
|
||||
Больше никаких прокси на contracts.kube5s.ru — всё локально.
|
||||
"""
|
||||
import os
|
||||
import httpx
|
||||
from flask import Flask, render_template, request, jsonify
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
VM_API = os.environ.get("VM_API", "https://contracts.kube5s.ru")
|
||||
LLM_URL = os.environ.get("LLM_API_URL", "https://api.aillm.ru/v1/chat/completions")
|
||||
LLM_KEY = os.environ.get("LLM_API_KEY", "")
|
||||
LLM_MODEL = os.environ.get("LLM_MODEL", "gpt-oss-120b")
|
||||
from flask import Flask
|
||||
from app.config import VERSION, MAX_CONTENT_LENGTH
|
||||
from app.routes import register_routes
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def index():
|
||||
return render_template("index.html", vm_api=VM_API)
|
||||
def create_app():
|
||||
app = Flask(__name__)
|
||||
app.config["VERSION"] = VERSION
|
||||
app.config["MAX_CONTENT_LENGTH"] = MAX_CONTENT_LENGTH
|
||||
|
||||
# DB auto-seed — schema migration + seed prompts + crash recovery
|
||||
_init_db()
|
||||
|
||||
# Регистрация всех blueprint'ов
|
||||
register_routes(app)
|
||||
|
||||
# no_cache на все ответы
|
||||
@app.after_request
|
||||
def no_cache(response):
|
||||
response.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
|
||||
response.headers["Pragma"] = "no-cache"
|
||||
response.headers["Expires"] = "0"
|
||||
return response
|
||||
|
||||
return app
|
||||
|
||||
|
||||
@app.route("/chat", methods=["POST"])
|
||||
def chat():
|
||||
contract_id = request.args.get("contract_id", "")
|
||||
question = request.form.get("question", "")
|
||||
if not contract_id or not question:
|
||||
return jsonify({"ok": False, "error": "contract_id and question required"})
|
||||
|
||||
def _init_db():
|
||||
"""Schema migration + recovery: сброс застрявших classify."""
|
||||
try:
|
||||
with httpx.Client(timeout=10) as client:
|
||||
resp = client.get(f"{VM_API}/api/spec-current", params={"contract_id": contract_id})
|
||||
resp.raise_for_status()
|
||||
rows = resp.json().get("rows", [])
|
||||
except Exception as e:
|
||||
return jsonify({"ok": False, "error": f"VM error: {e}"})
|
||||
from app.db.connection import execute
|
||||
|
||||
if not rows:
|
||||
return jsonify({"ok": True, "answer": "Нет данных. Сначала запустите сравнение."})
|
||||
for sql in [
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_raw text",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_input text",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS zip_source text",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS content_hash text",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS batch_id uuid",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_status text DEFAULT 'pending'",
|
||||
"ALTER TABLE documents ALTER COLUMN classify_status SET DEFAULT 'pending'",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS doc_type text",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS own_number text",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS parent_number text",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS doc_date date",
|
||||
"ALTER TABLE documents ADD COLUMN IF NOT EXISTS counterparty text",
|
||||
]:
|
||||
execute(sql)
|
||||
|
||||
ctx = "\n".join(f"{r.get('name','')} | цена={r.get('price')} | объём={r.get('qty')} | сумма={r.get('sum')} | начало={r.get('date_start')}" for r in rows)
|
||||
prompt = f"Ты — анализатор договоров. Данные:\n{ctx}\n\nВопрос: {question}\nОтветь кратко, только по данным."
|
||||
from app.db import prompts as db_prompts
|
||||
db_prompts.seed_defaults()
|
||||
|
||||
try:
|
||||
with httpx.Client(http2=True, timeout=120) as client:
|
||||
resp = client.post(LLM_URL, json={
|
||||
"model": LLM_MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": 1000, "temperature": 0.1,
|
||||
}, headers={"Authorization": f"Bearer {LLM_KEY}", "Content-Type": "application/json"})
|
||||
resp.raise_for_status()
|
||||
return jsonify({"ok": True, "answer": resp.json()["choices"][0]["message"]["content"]})
|
||||
except Exception as e:
|
||||
return jsonify({"ok": False, "error": f"LLM error: {e}"})
|
||||
# Crash recovery: сбросить застрявшие classify
|
||||
execute(
|
||||
"UPDATE documents SET classify_status='pending', error_message=NULL "
|
||||
"WHERE classify_status='processing'"
|
||||
)
|
||||
except Exception:
|
||||
pass # БД недоступна при старте — не фатально
|
||||
|
||||
|
||||
@app.route("/api/prompts", methods=["GET"])
|
||||
def prompts_get():
|
||||
try:
|
||||
with httpx.Client(timeout=10) as client:
|
||||
resp = client.get(f"{VM_API}/api/prompts", params=request.args)
|
||||
return jsonify(resp.json())
|
||||
except Exception as e:
|
||||
return jsonify({"ok": False, "error": str(e)})
|
||||
|
||||
|
||||
@app.route("/api/prompts/list", methods=["GET"])
|
||||
def prompts_list():
|
||||
try:
|
||||
with httpx.Client(timeout=10) as client:
|
||||
resp = client.get(f"{VM_API}/api/prompts/list")
|
||||
return jsonify(resp.json())
|
||||
except Exception as e:
|
||||
return jsonify({"ok": False, "error": str(e)})
|
||||
|
||||
|
||||
@app.route("/api/prompts/save", methods=["POST"])
|
||||
def prompts_save():
|
||||
try:
|
||||
with httpx.Client(timeout=10) as client:
|
||||
resp = client.post(f"{VM_API}/api/prompts/save", json=request.get_json())
|
||||
return jsonify(resp.json())
|
||||
except Exception as e:
|
||||
return jsonify({"ok": False, "error": str(e)})
|
||||
|
||||
|
||||
@app.route("/api/prompts/activate", methods=["POST"])
|
||||
def prompts_activate():
|
||||
try:
|
||||
with httpx.Client(timeout=10) as client:
|
||||
resp = client.post(f"{VM_API}/api/prompts/activate", json=request.get_json())
|
||||
return jsonify(resp.json())
|
||||
except Exception as e:
|
||||
return jsonify({"ok": False, "error": str(e)})
|
||||
|
||||
|
||||
@app.route("/architect")
|
||||
def architect():
|
||||
return render_template("architect.html")
|
||||
|
||||
|
||||
@app.route("/ci-cd")
|
||||
def ci_cd():
|
||||
return render_template("ci-cd.html")
|
||||
|
||||
|
||||
@app.route("/drhider")
|
||||
@app.route("/DrHider")
|
||||
def drhider():
|
||||
return render_template("drhider.html", vm_api=VM_API)
|
||||
|
||||
|
||||
@app.route("/health")
|
||||
def health():
|
||||
return {"ok": True, "service": "contracts-flask"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host="0.0.0.0", port=5000, threaded=True)
|
||||
|
||||
# ── Совместимость с Lucee-путями ──────────────────────────────
|
||||
|
||||
@app.route("/chat.cfm", methods=["POST"])
|
||||
def chat_cfm():
|
||||
return chat()
|
||||
|
||||
@app.route("/prompt.cfm", methods=["GET"])
|
||||
def prompt_cfm():
|
||||
return prompts_get()
|
||||
app = create_app()
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Конфигурация приложения — все настройки в одном месте."""
|
||||
import os
|
||||
|
||||
VERSION = "2.0.0"
|
||||
|
||||
DB_CONFIG = {
|
||||
"host": os.getenv("DB_HOST", "127.0.0.1"),
|
||||
"port": int(os.getenv("DB_PORT", "5432")),
|
||||
"dbname": os.getenv("DB_NAME", "baza"),
|
||||
"user": os.getenv("DB_USER", "super"),
|
||||
"password": os.getenv("DB_PASS", ""),
|
||||
}
|
||||
|
||||
LLM_URL = os.getenv("LLM_API_URL", "https://api.aillm.ru/v1/chat/completions")
|
||||
LLM_KEY = os.getenv("LLM_API_KEY", "")
|
||||
LLM_MODEL = os.getenv("LLM_MODEL", "gpt-oss-120b")
|
||||
|
||||
MAX_CONTENT_LENGTH = 200 * 1024 * 1024 # 200 MB
|
||||
API_KEY = os.getenv("API_KEY", "")
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Регистрация всех blueprint'ов."""
|
||||
|
||||
|
||||
def register_routes(app):
|
||||
from app.routes.upload_bp import upload_bp
|
||||
from app.routes.pipeline_bp import pipeline_bp
|
||||
from app.routes.api_bp import api_bp
|
||||
from app.routes.prompts_bp import prompts_bp
|
||||
from app.routes.health_bp import health_bp
|
||||
from app.routes.pages_bp import pages_bp
|
||||
|
||||
app.register_blueprint(upload_bp)
|
||||
app.register_blueprint(pipeline_bp)
|
||||
app.register_blueprint(api_bp)
|
||||
app.register_blueprint(prompts_bp)
|
||||
app.register_blueprint(health_bp)
|
||||
app.register_blueprint(pages_bp)
|
||||
@@ -0,0 +1,150 @@
|
||||
"""API blueprint — groups, documents, supplements, sync, cleanup, spec-current."""
|
||||
from flask import Blueprint, request, jsonify
|
||||
from app.db import documents, supplements, spec_current
|
||||
from app.db.connection import execute, query
|
||||
from app.services.grouping import group_documents, apply_groups
|
||||
from app.config import API_KEY
|
||||
|
||||
api_bp = Blueprint("api", __name__)
|
||||
|
||||
|
||||
def _require_auth():
|
||||
"""Проверка API-ключа для опасных операций."""
|
||||
if API_KEY:
|
||||
key = request.headers.get("X-Api-Key", "")
|
||||
if key != API_KEY:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# ── Supplements ──────────────────────────────────────────────────
|
||||
|
||||
@api_bp.route("/api/supplements")
|
||||
def api_supplements():
|
||||
cid = request.args.get("contract_id")
|
||||
if not cid:
|
||||
return jsonify(ok=False, error="contract_id required"), 400
|
||||
rows = supplements.list_by_contract(cid)
|
||||
return jsonify(ok=True, supplements=rows)
|
||||
|
||||
|
||||
# ── Documents ────────────────────────────────────────────────────
|
||||
|
||||
@api_bp.route("/api/documents/<doc_id>")
|
||||
def api_document(doc_id):
|
||||
doc = documents.get(doc_id)
|
||||
if not doc:
|
||||
return jsonify(ok=False, error="not found"), 404
|
||||
return jsonify(ok=True, **{
|
||||
k: doc.get(k) for k in [
|
||||
"id", "filename", "status", "elements_json", "doc_type",
|
||||
"own_number", "parent_number", "doc_date", "counterparty",
|
||||
"classify_status", "classify_raw", "classify_input",
|
||||
]
|
||||
})
|
||||
|
||||
|
||||
@api_bp.route("/api/documents/<doc_id>", methods=["DELETE"])
|
||||
def api_document_delete(doc_id):
|
||||
supps = query("SELECT id, contract_id FROM supplements WHERE document_id = %s", (doc_id,))
|
||||
for s in (supps or []):
|
||||
execute(
|
||||
"""DELETE FROM spec_current WHERE contract_id = %s
|
||||
AND last_event_id IN (SELECT id FROM spec_events WHERE supplement_id = %s)""",
|
||||
(s["contract_id"], s["id"]),
|
||||
)
|
||||
execute("DELETE FROM spec_events WHERE supplement_id = %s", (s["id"],))
|
||||
execute("DELETE FROM supplements WHERE id = %s", (s["id"],))
|
||||
execute("DELETE FROM documents WHERE id = %s", (doc_id,))
|
||||
return jsonify(ok=True)
|
||||
|
||||
|
||||
# ── Sync ─────────────────────────────────────────────────────────
|
||||
|
||||
@api_bp.route("/api/sync", methods=["POST"])
|
||||
def api_sync():
|
||||
"""Удалить документы, не входящие в keep_ids."""
|
||||
body = request.get_json()
|
||||
keep_ids = set(body.get("keep_ids", []))
|
||||
if len(keep_ids) > 1000:
|
||||
return jsonify(ok=False, error="too many keep_ids (max 1000)"), 400
|
||||
|
||||
docs = query("SELECT id FROM documents", ())
|
||||
deleted = 0
|
||||
for d in (docs or []):
|
||||
if d["id"] in keep_ids:
|
||||
continue
|
||||
supps = query("SELECT id, contract_id FROM supplements WHERE document_id = %s", (d["id"],))
|
||||
for s in (supps or []):
|
||||
execute(
|
||||
"""DELETE FROM spec_current WHERE contract_id = %s
|
||||
AND last_event_id IN (SELECT id FROM spec_events WHERE supplement_id = %s)""",
|
||||
(s["contract_id"], s["id"]),
|
||||
)
|
||||
execute("DELETE FROM spec_events WHERE supplement_id = %s", (s["id"],))
|
||||
execute("DELETE FROM supplements WHERE id = %s", (s["id"],))
|
||||
execute("DELETE FROM documents WHERE id = %s", (d["id"],))
|
||||
deleted += 1
|
||||
|
||||
execute("DELETE FROM contracts WHERE id NOT IN (SELECT DISTINCT contract_id FROM supplements)")
|
||||
return jsonify(ok=True, deleted=deleted)
|
||||
|
||||
|
||||
# ── Groups ───────────────────────────────────────────────────────
|
||||
|
||||
@api_bp.route("/api/groups")
|
||||
def api_groups():
|
||||
batch_id = request.args.get("batch")
|
||||
if not batch_id:
|
||||
return jsonify(ok=False, error="batch required"), 400
|
||||
result = group_documents(batch_id)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@api_bp.route("/api/batch-progress")
|
||||
def api_batch_progress():
|
||||
batch_id = request.args.get("batch")
|
||||
if not batch_id:
|
||||
return jsonify(ok=False, error="batch required"), 400
|
||||
counts = documents.count_by_status(batch_id)
|
||||
docs = documents.list_by_batch(batch_id)
|
||||
return jsonify(ok=True, counts=counts, total=len(docs))
|
||||
|
||||
|
||||
@api_bp.route("/api/apply-groups", methods=["POST"])
|
||||
def api_apply_groups():
|
||||
body = request.get_json()
|
||||
batch_id = body.get("batch_id")
|
||||
groups = body.get("groups", [])
|
||||
if not batch_id:
|
||||
return jsonify(ok=False, error="batch_id required"), 400
|
||||
result = apply_groups(batch_id, groups)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# ── Spec current ─────────────────────────────────────────────────
|
||||
|
||||
@api_bp.route("/api/spec-current")
|
||||
def api_spec_current():
|
||||
cid = request.args.get("contract_id")
|
||||
if not cid:
|
||||
return jsonify(ok=False, error="contract_id required"), 400
|
||||
rows = spec_current.list_by_contract(cid)
|
||||
return jsonify(ok=True, rows=rows)
|
||||
|
||||
|
||||
# ── Cleanup (требует API-ключ) ────────────────────────────────────
|
||||
|
||||
@api_bp.route("/api/cleanup", methods=["POST"])
|
||||
def api_cleanup():
|
||||
"""Полная очистка всех данных (кроме prompts)."""
|
||||
if not _require_auth():
|
||||
return jsonify(ok=False, error="unauthorized"), 401
|
||||
|
||||
execute("DELETE FROM spec_current")
|
||||
execute("DELETE FROM spec_events")
|
||||
execute("DELETE FROM supplements")
|
||||
execute("DELETE FROM upload_chunks")
|
||||
execute("DELETE FROM documents")
|
||||
execute("DELETE FROM contracts")
|
||||
return jsonify(ok=True, message="all data cleaned")
|
||||
@@ -0,0 +1,10 @@
|
||||
"""Health probe — обязательно для Штурвала."""
|
||||
from flask import Blueprint, jsonify
|
||||
from app.config import VERSION
|
||||
|
||||
health_bp = Blueprint("health", __name__)
|
||||
|
||||
|
||||
@health_bp.route("/health")
|
||||
def health():
|
||||
return jsonify({"ok": True, "version": VERSION})
|
||||
@@ -0,0 +1,14 @@
|
||||
"""HTML-страницы."""
|
||||
from flask import Blueprint, render_template
|
||||
|
||||
pages_bp = Blueprint("pages", __name__)
|
||||
|
||||
|
||||
@pages_bp.route("/")
|
||||
def index():
|
||||
return render_template("index.html")
|
||||
|
||||
|
||||
@pages_bp.route("/architect")
|
||||
def architect():
|
||||
return render_template("architect.html")
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Pipeline blueprint — SSE-сравнение + classify."""
|
||||
import json, re, os, threading
|
||||
from flask import Blueprint, request, jsonify, Response, stream_with_context
|
||||
from app.services.process import run_pipeline
|
||||
from app.services.classify import classify_batch
|
||||
from app.llm_prompt import build_prompt
|
||||
from app.db import documents
|
||||
|
||||
pipeline_bp = Blueprint("pipeline", __name__)
|
||||
|
||||
# In-memory lock для classify (замена файлового lock)
|
||||
_classify_locks: dict[str, threading.Thread] = {}
|
||||
|
||||
|
||||
@pipeline_bp.route("/process-v2", methods=["GET"])
|
||||
def process_v2():
|
||||
"""SSE-стриминг сравнения договоров."""
|
||||
cid = request.args.get("contract_id")
|
||||
if not cid or not re.fullmatch(
|
||||
r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', cid, re.I
|
||||
):
|
||||
return jsonify(ok=False, error="invalid contract_id"), 400
|
||||
|
||||
order_ids = request.args.get("order", "")
|
||||
|
||||
def generate():
|
||||
# Heartbeat каждые 15 сек — держит соединение при медленном LLM
|
||||
import time as _time
|
||||
last_beat = _time.time()
|
||||
|
||||
yield ": ok\n\n"
|
||||
try:
|
||||
for event in run_pipeline(cid, order_ids, build_prompt):
|
||||
now = _time.time()
|
||||
if now - last_beat >= 15:
|
||||
yield ": heartbeat\n\n"
|
||||
last_beat = now
|
||||
yield f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
|
||||
except GeneratorExit:
|
||||
return
|
||||
except Exception as e:
|
||||
yield f"data: {json.dumps({'type': 'error', 'message': str(e)}, ensure_ascii=False)}\n\n"
|
||||
|
||||
return Response(
|
||||
stream_with_context(generate()),
|
||||
content_type="text/event-stream; charset=utf-8",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pipeline_bp.route("/api/classify-batch", methods=["POST"])
|
||||
def classify_batch_route():
|
||||
"""Запустить классификацию. ≤10 файлов — sync, >10 — async (Thread)."""
|
||||
body = request.get_json()
|
||||
batch_id = body.get("batch_id")
|
||||
if not batch_id:
|
||||
return jsonify(ok=False, error="batch_id required"), 400
|
||||
|
||||
# Guard: уже запущена?
|
||||
if batch_id in _classify_locks:
|
||||
t = _classify_locks[batch_id]
|
||||
if t.is_alive():
|
||||
return jsonify(ok=False, error="classify already running"), 409
|
||||
else:
|
||||
del _classify_locks[batch_id]
|
||||
|
||||
pending = documents.list_pending(batch_id)
|
||||
total = len(pending)
|
||||
if total == 0:
|
||||
return jsonify(ok=False, error="no pending documents"), 400
|
||||
|
||||
# Sync для малых батчей
|
||||
if total <= 10:
|
||||
result = classify_batch(batch_id)
|
||||
return jsonify(result)
|
||||
|
||||
# Async для больших
|
||||
def _run():
|
||||
try:
|
||||
classify_batch(batch_id)
|
||||
finally:
|
||||
_classify_locks.pop(batch_id, None)
|
||||
|
||||
t = threading.Thread(target=_run, daemon=True)
|
||||
_classify_locks[batch_id] = t
|
||||
t.start()
|
||||
|
||||
return jsonify(ok=True, total=total), 202
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Prompts blueprint — CRUD + activate для версионирования промптов."""
|
||||
from flask import Blueprint, request, jsonify
|
||||
from app.db import prompts as db_prompts
|
||||
|
||||
prompts_bp = Blueprint("prompts", __name__)
|
||||
|
||||
|
||||
@prompts_bp.route("/api/prompts", methods=["GET"])
|
||||
def prompts_get():
|
||||
role = request.args.get("role")
|
||||
if not role:
|
||||
return jsonify(ok=False, error="role required"), 400
|
||||
p = db_prompts.get_active(role)
|
||||
if p:
|
||||
return jsonify(ok=True, **{
|
||||
"id": p["id"], "role": p["role"], "name": p["name"],
|
||||
"body": p["body"], "is_active": p["is_active"],
|
||||
"notes": p.get("notes"), "created_at": str(p.get("created_at", "")),
|
||||
})
|
||||
return jsonify(ok=False, error="not found"), 404
|
||||
|
||||
|
||||
@prompts_bp.route("/api/prompts/list", methods=["GET"])
|
||||
def prompts_list():
|
||||
role = request.args.get("role")
|
||||
if not role:
|
||||
return jsonify(ok=False, error="role required"), 400
|
||||
versions = db_prompts.list_by_role(role)
|
||||
return jsonify(ok=True, versions=versions)
|
||||
|
||||
|
||||
@prompts_bp.route("/api/prompts/save", methods=["POST"])
|
||||
def prompts_save():
|
||||
body = request.get_json()
|
||||
if not body:
|
||||
return jsonify(ok=False, error="body required"), 400
|
||||
role = body.get("role", "")
|
||||
name = body.get("name", "")
|
||||
prompt_body = body.get("body", "")
|
||||
notes = body.get("notes", "")
|
||||
try:
|
||||
p = db_prompts.insert(role, name, prompt_body, notes)
|
||||
return jsonify(ok=True, id=p["id"])
|
||||
except Exception as e:
|
||||
return jsonify(ok=False, error=str(e)), 500
|
||||
|
||||
|
||||
@prompts_bp.route("/api/prompts/activate", methods=["POST"])
|
||||
def prompts_activate():
|
||||
body = request.get_json()
|
||||
if not body:
|
||||
return jsonify(ok=False, error="body required"), 400
|
||||
prompt_id = body.get("id")
|
||||
try:
|
||||
db_prompts.activate(prompt_id)
|
||||
return jsonify(ok=True)
|
||||
except Exception as e:
|
||||
return jsonify(ok=False, error=str(e)), 500
|
||||
|
||||
|
||||
@prompts_bp.route("/api/prompts/delete", methods=["POST"])
|
||||
def prompts_delete():
|
||||
body = request.get_json()
|
||||
if not body:
|
||||
return jsonify(ok=False, error="body required"), 400
|
||||
prompt_id = body.get("id")
|
||||
try:
|
||||
db_prompts.delete(prompt_id)
|
||||
return jsonify(ok=True)
|
||||
except Exception as e:
|
||||
return jsonify(ok=False, error=str(e)), 500
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Upload blueprint — загрузка, конвертация, распаковка."""
|
||||
import io, os, base64, hashlib, zipfile, tempfile, subprocess
|
||||
from flask import Blueprint, request, jsonify, send_file
|
||||
from app.services.parse import parse_file
|
||||
from app.db import documents
|
||||
from app.config import MAX_CONTENT_LENGTH
|
||||
|
||||
upload_bp = Blueprint("upload", __name__)
|
||||
|
||||
ALLOWED = {"pdf", "docx", "doc", "zip"}
|
||||
|
||||
|
||||
def _check_ext(filename: str) -> str | None:
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
if ext not in ALLOWED:
|
||||
return f"unsupported format: .{ext} (allowed: {', '.join(sorted(ALLOWED))})"
|
||||
return None
|
||||
|
||||
|
||||
@upload_bp.route("/upload", methods=["POST"])
|
||||
def upload():
|
||||
"""Загрузка одного файла + авто-парсинг → БД."""
|
||||
f = request.files.get("files")
|
||||
if not f:
|
||||
return jsonify(ok=False, error="no file"), 400
|
||||
|
||||
err = _check_ext(f.filename)
|
||||
if err:
|
||||
return jsonify(ok=False, error=err), 400
|
||||
|
||||
data = f.read()
|
||||
content_hash = hashlib.sha256(data).hexdigest()[:16]
|
||||
batch_id = request.form.get("batch_id")
|
||||
zip_source = request.form.get("zip_source")
|
||||
|
||||
# Дедупликация по хешу
|
||||
if batch_id:
|
||||
existing = documents.get_by_hash(batch_id, content_hash)
|
||||
if existing:
|
||||
return jsonify(ok=False, error="duplicate", doc_id=existing["id"])
|
||||
|
||||
doc = documents.insert(
|
||||
filename=f.filename,
|
||||
mime_type=f.content_type or "application/octet-stream",
|
||||
original_bytes=base64.b64encode(data).decode(),
|
||||
batch_id=batch_id,
|
||||
zip_source=zip_source,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
|
||||
# Авто-парсинг
|
||||
try:
|
||||
result = parse_file(f.filename, data)
|
||||
if result["status"] == "parsed":
|
||||
documents.set_parsed(doc["id"], result["elements"])
|
||||
else:
|
||||
documents.set_error(doc["id"], result.get("error", "parse failed"))
|
||||
except Exception as e:
|
||||
documents.set_error(doc["id"], str(e))
|
||||
result = {"status": "error", "error": str(e)}
|
||||
|
||||
contract_id = request.form.get("contract_id")
|
||||
return jsonify(
|
||||
ok=True,
|
||||
doc_id=doc["id"],
|
||||
contract_id=contract_id,
|
||||
parsed={"status": result["status"], "element_count": result.get("element_count", 0)},
|
||||
)
|
||||
|
||||
|
||||
@upload_bp.route("/convert-doc", methods=["POST"])
|
||||
def convert_doc():
|
||||
""".doc → .docx через libreoffice (без сохранения на диск)."""
|
||||
f = request.files.get("files")
|
||||
if not f:
|
||||
return jsonify(ok=False, error="no file"), 400
|
||||
|
||||
data = f.read()
|
||||
doc_path = None
|
||||
tmpdir = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as tmp:
|
||||
tmp.write(data)
|
||||
doc_path = tmp.name
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
subprocess.run(
|
||||
["libreoffice", "--headless", "--convert-to", "docx", "--outdir", tmpdir, doc_path],
|
||||
timeout=30, capture_output=True,
|
||||
)
|
||||
docx_files = [x for x in os.listdir(tmpdir) if x.endswith(".docx")]
|
||||
if docx_files:
|
||||
with open(os.path.join(tmpdir, docx_files[0]), "rb") as out:
|
||||
return send_file(
|
||||
io.BytesIO(out.read()),
|
||||
mimetype="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
)
|
||||
return jsonify(ok=False, error="conversion produced no output"), 500
|
||||
finally:
|
||||
if doc_path and os.path.exists(doc_path):
|
||||
os.unlink(doc_path)
|
||||
if tmpdir and os.path.exists(tmpdir):
|
||||
for x in os.listdir(tmpdir):
|
||||
os.unlink(os.path.join(tmpdir, x))
|
||||
os.rmdir(tmpdir)
|
||||
|
||||
|
||||
@upload_bp.route("/unzip-upload", methods=["POST"])
|
||||
def unzip_upload():
|
||||
"""Распаковать ZIP → список файлов (base64 для фронтенда)."""
|
||||
f = request.files.get("files")
|
||||
if not f:
|
||||
return jsonify(ok=False, error="no file"), 400
|
||||
|
||||
data = f.read()
|
||||
MAX_FILES = 500
|
||||
MAX_UNCOMPRESSED = 500 * 1024 * 1024 # 500 MB
|
||||
|
||||
files = []
|
||||
total = 0
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
||||
if len(zf.namelist()) > MAX_FILES:
|
||||
return jsonify(ok=False, error=f"too many files in ZIP (max {MAX_FILES})"), 400
|
||||
|
||||
for info in zf.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
name = os.path.basename(info.filename)
|
||||
if not name or ".." in name or "/" in name or "\\" in name:
|
||||
continue
|
||||
|
||||
raw = zf.read(info)
|
||||
total += len(raw)
|
||||
if total > MAX_UNCOMPRESSED:
|
||||
return jsonify(ok=False, error="total uncompressed size exceeds 500 MB"), 400
|
||||
|
||||
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
|
||||
files.append({
|
||||
"filename": name,
|
||||
"ext": ext,
|
||||
"size": len(raw),
|
||||
"data_b64": base64.b64encode(raw).decode(),
|
||||
})
|
||||
|
||||
return jsonify(ok=True, files=files)
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Process service — SSE pipeline: reset → supplements → LLM → apply.
|
||||
Адаптирован из deploy/compare/process.py: callback → generator.
|
||||
"""
|
||||
import json, time
|
||||
from app.db import supplements, spec_current, spec_events
|
||||
from app.services.metrics import check_arithmetic
|
||||
|
||||
|
||||
def run_pipeline(contract_id, order_ids, build_prompt_fn):
|
||||
"""Generator: yield SSE events вместо sse_send callback.
|
||||
|
||||
Использование:
|
||||
for event in run_pipeline(cid, order_ids, build_prompt):
|
||||
yield f"data: {json.dumps(event)}\\n\\n"
|
||||
"""
|
||||
t0 = time.time()
|
||||
|
||||
# 0. Reset
|
||||
spec_events.reset(contract_id)
|
||||
|
||||
# 1. Get supplements with parsed documents
|
||||
supps = supplements.list_by_contract(contract_id)
|
||||
if order_ids:
|
||||
order_list = [x.strip() for x in order_ids.split(",") if x.strip()]
|
||||
order_map = {oid: i for i, oid in enumerate(order_list)}
|
||||
supps.sort(key=lambda s: order_map.get(s["id"], 999999))
|
||||
else:
|
||||
supps.sort(key=lambda s: (s.get("doc_date") or "9999-99-99", s.get("created_at", "")))
|
||||
|
||||
if not supps:
|
||||
yield {"type": "error", "message": "Нет распарсенных файлов"}
|
||||
return
|
||||
|
||||
from app.services.llm import call_llm
|
||||
|
||||
for s in supps:
|
||||
sid = s["id"]
|
||||
filename = s.get("filename", "?")
|
||||
|
||||
# Current spec
|
||||
cur = spec_current.list_by_contract(contract_id)
|
||||
current_spec = []
|
||||
for r in cur:
|
||||
current_spec.append({
|
||||
"hash": r["name_hash"],
|
||||
"name": r["name"],
|
||||
"price": float(r["price"]) if r.get("price") is not None else None,
|
||||
"qty": float(r["qty"]) if r.get("qty") is not None else None,
|
||||
"sum": float(r["sum"]) if r.get("sum") is not None else None,
|
||||
"date_start": r["date_start"],
|
||||
})
|
||||
|
||||
# Get elements_json
|
||||
ej = spec_current.get_elements_json(s["document_id"])
|
||||
if not ej:
|
||||
yield {
|
||||
"type": "extract_error",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
"error": "no elements_json",
|
||||
}
|
||||
continue
|
||||
|
||||
# Build doc text from elements
|
||||
doc_text = _elements_to_text(ej)
|
||||
|
||||
yield {
|
||||
"type": "extract_start",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
}
|
||||
|
||||
# LLM call
|
||||
try:
|
||||
t1 = time.time()
|
||||
result, prompt_id = call_llm(current_spec, doc_text, build_prompt_fn)
|
||||
ops = result.get("ops", [])
|
||||
mode = result.get("mode", "llm")
|
||||
|
||||
yield {
|
||||
"type": "llm_done",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
"ops_count": len(ops),
|
||||
"mode": mode,
|
||||
"time_s": round(time.time() - t1, 1),
|
||||
}
|
||||
|
||||
# Apply ops to DB
|
||||
applied_ops = []
|
||||
summary = {"added": 0, "updated": 0, "deleted": 0, "unresolved": 0}
|
||||
for op in ops:
|
||||
action = op.get("action", "UNRESOLVED")
|
||||
summary[action.lower()] = summary.get(action.lower(), 0) + 1
|
||||
try:
|
||||
if action == "ADD":
|
||||
nr = op.get("new_row", {})
|
||||
spec_events.add_row(contract_id, sid, nr)
|
||||
elif action == "UPDATE":
|
||||
nr = op.get("new_row", {})
|
||||
nv = op.get("new_values", {})
|
||||
target = op.get("target_hash", "")
|
||||
spec_events.update_row(contract_id, sid, target, nr, nv)
|
||||
elif action == "DELETE":
|
||||
target = op.get("target_hash", "")
|
||||
spec_events.delete_row(contract_id, sid, target)
|
||||
applied_ops.append(op)
|
||||
except Exception:
|
||||
summary["unresolved"] = summary.get("unresolved", 0) + 1
|
||||
|
||||
# Arithmetic check
|
||||
check_arithmetic(contract_id)
|
||||
|
||||
yield {
|
||||
"type": "applied",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
"summary": summary,
|
||||
"ops": applied_ops,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
yield {
|
||||
"type": "extract_error",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
total_time = round(time.time() - t0, 1)
|
||||
yield {"type": "complete", "total_time_s": total_time}
|
||||
|
||||
|
||||
def _elements_to_text(ej):
|
||||
"""Extract flat text from elements_json for LLM prompt."""
|
||||
if isinstance(ej, dict) and "Value" in ej:
|
||||
ej = ej["Value"]
|
||||
if isinstance(ej, str):
|
||||
try:
|
||||
ej = json.loads(ej)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return ej
|
||||
|
||||
lines = []
|
||||
if isinstance(ej, list):
|
||||
for el in ej:
|
||||
if isinstance(el, dict):
|
||||
if el.get("type") == "paragraph":
|
||||
lines.append(el.get("text", ""))
|
||||
elif el.get("type") == "table":
|
||||
for row in el.get("rows", []):
|
||||
lines.append(" | ".join(str(c) for c in row))
|
||||
elif isinstance(el, str):
|
||||
lines.append(el)
|
||||
return "\n".join(lines)
|
||||
+6
-7
@@ -1,10 +1,9 @@
|
||||
// ⛔ НЕ МЕНЯТЬ БЕЗ РАЗРЕШЕНИЯ НАЕЛЯ ⛔
|
||||
// Contracts App — весь JS на VM (contracts.kube5s.ru)
|
||||
// Lucee: только домен + index.cfm-скелет
|
||||
var VM_API = 'https://check.kube5s.ru';
|
||||
var UPLOAD_URL = VM_API + '/upload';
|
||||
var CONVERT_URL = VM_API + '/convert-doc';
|
||||
var UNZIP_URL = VM_API + '/unzip-upload';
|
||||
// Contracts App v2.0 — всё на Flask, ВМ больше нет
|
||||
var VM_API = '';
|
||||
var UPLOAD_URL = '/upload';
|
||||
var CONVERT_URL = '/convert-doc';
|
||||
var UNZIP_URL = '/unzip-upload';
|
||||
// SITE_URL удалён (Фаза 4) — не использовался
|
||||
|
||||
var fileInput = document.getElementById('fileInput');
|
||||
@@ -328,7 +327,7 @@ document.getElementById('llmBtn').addEventListener('click', function() {
|
||||
compareStatus.textContent = '⏳ 0.0с';
|
||||
|
||||
var order = state.files.map(function(f) { return f.supp_id; }).filter(Boolean).join(',');
|
||||
var url = 'https://check.kube5s.ru/process-v2?contract_id=' + state.contractId +
|
||||
var url = '/process-v2?contract_id=' + state.contractId +
|
||||
(order ? '&order=' + encodeURIComponent(order) : '');
|
||||
|
||||
// Фаза 3: унифицированный SSE через startCompareSSE (compare.js)
|
||||
|
||||
@@ -73,7 +73,7 @@
|
||||
<body>
|
||||
<div class="topbar">
|
||||
<img src="/static/logo.svg" alt="Nubes">
|
||||
<span class="title">Сверка договоров — LLM AI-driven Event Sourcing <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.0.196-flask</span></span>
|
||||
<span class="title">Сверка договоров — LLM AI-driven Event Sourcing <span style="font-weight:400;color:var(--muted);font-size:12px;">v2.0.0</span></span>
|
||||
<div id="pipelineStepper" style="display:flex;gap:8px;font-size:11px;align-items:center;color:var(--muted);">
|
||||
<span id="stepUpload">○ Загрузка</span><span>→</span>
|
||||
<span id="stepClassify">○ Классификация</span><span>→</span>
|
||||
|
||||
@@ -3,3 +3,4 @@ gunicorn
|
||||
httpx
|
||||
python-docx
|
||||
pdfplumber
|
||||
psycopg2-binary
|
||||
|
||||
Reference in New Issue
Block a user