v1.0.155: VM PostgreSQL + modular db/ & services/ — Lucee stays SPA shell only

deploy/services/llm.py

@@ -0,0 +1,37 @@
+"""LLM service — call LLM API."""
+import os, json
+import httpx
+
+LLM_URL = "https://api.aillm.ru/v1/chat/completions"
+LLM_KEY = os.environ.get("LLM_KEY", "")
+LLM_MODEL = "gpt-oss-120b"
+
+
+def call_llm(current_spec, doc_text, build_prompt_fn):
+    """Call LLM. Returns (parsed_result, prompt_id)."""
+    prompt, prompt_id = build_prompt_fn(current_spec, doc_text)
+    payload = {
+        "model": LLM_MODEL,
+        "messages": [{"role": "user", "content": prompt}],
+        "max_tokens": 8000,
+        "temperature": 0.1,
+    }
+    with httpx.Client(http2=True, timeout=120, verify=False) as client:
+        resp = client.post(
+            LLM_URL,
+            json=payload,
+            headers={
+                "Authorization": f"Bearer {LLM_KEY}",
+                "Content-Type": "application/json",
+            },
+        )
+        resp.raise_for_status()
+        data = resp.json()
+
+    raw_text = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+    json_text = raw_text
+    if "```json" in json_text:
+        json_text = json_text.split("```json")[1].split("```")[0]
+    elif "```" in json_text:
+        json_text = json_text.split("```")[1].split("```")[0]
+    return json.loads(json_text.strip()), prompt_id

deploy/services/parse.py

@@ -0,0 +1,73 @@
+"""Parse service — PDF (PyPDF2), DOCX (python-docx), plain text fallback."""
+import io
+from PyPDF2 import PdfReader
+
+
+def parse_file(filename, data):
+    """Parse file bytes → {status, elements, element_count}."""
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    try:
+        if ext == "pdf":
+            return _parse_pdf(data)
+        elif ext == "docx":
+            return _parse_docx(data)
+        elif ext == "doc":
+            return _parse_docx(data)  # python-docx handles most .doc
+        else:
+            return _parse_text(data)
+    except Exception as e:
+        return {"status": "error", "error": str(e), "element_count": 0}
+
+
+def _parse_pdf(data):
+    reader = PdfReader(io.BytesIO(data))
+    elements = []
+    for page in reader.pages:
+        text = page.extract_text()
+        if text:
+            for line in text.split("\n"):
+                line = line.strip()
+                if line:
+                    elements.append({"type": "paragraph", "text": line, "style": ""})
+    return {"status": "parsed", "element_count": len(elements), "elements": elements}
+
+
+def _parse_docx(data):
+    import docx
+    doc = docx.Document(io.BytesIO(data))
+    elements = []
+
+    for block in doc.element.body:
+        tag = block.tag.split("}")[-1] if "}" in block.tag else block.tag
+        if tag == "p":
+            text = _extract_paragraph_text(block)
+            if text:
+                elements.append({"type": "paragraph", "text": text, "style": ""})
+        elif tag == "tbl":
+            rows = []
+            for tr in block.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tr"):
+                cells = []
+                for tc in tr.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tc"):
+                    cell_text = "".join(t.text or "" for t in tc.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t"))
+                    cells.append(cell_text.strip())
+                if cells:
+                    rows.append(cells)
+            if rows:
+                elements.append({"type": "table", "rows": rows})
+
+    return {"status": "parsed", "element_count": len(elements), "elements": elements}
+
+
+def _extract_paragraph_text(p_elem):
+    texts = []
+    for t in p_elem.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t"):
+        if t.text:
+            texts.append(t.text)
+    return "".join(texts).strip()
+
+
+def _parse_text(data):
+    text = data.decode("utf-8", errors="ignore")[:5000]
+    lines = [l.strip() for l in text.split("\n") if l.strip()]
+    return {"status": "parsed", "element_count": len(lines),
+            "elements": [{"type": "paragraph", "text": l, "style": ""} for l in lines]}

deploy/services/process.py

@@ -0,0 +1,132 @@
+"""Process service — SSE pipeline: reset → supplements → LLM → apply."""
+import json, time, threading
+from db import supplements, spec_current, spec_events
+
+
+def run_pipeline(contract_id, order_ids, sse_send, build_prompt_fn):
+    """Run full SSE comparison pipeline. sse_send is a callback(dict)."""
+    t0 = time.time()
+
+    # 0. Reset
+    spec_events.reset(contract_id)
+
+    # 1. Get supplements with parsed documents
+    supps = supplements.list_by_contract(contract_id)
+    if order_ids:
+        order_list = [x.strip() for x in order_ids.split(",") if x.strip()]
+        order_map = {oid: i for i, oid in enumerate(order_list)}
+        supps.sort(key=lambda s: order_map.get(s["id"], 999999))
+
+    if not supps:
+        sse_send({"type": "error", "message": "Нет распарсенных файлов"})
+        return
+
+    from .llm import call_llm
+
+    for s in supps:
+        sid = s["id"]
+
+        # Current spec
+        cur = spec_current.list_by_contract(contract_id)
+        current_spec = []
+        for r in cur:
+            current_spec.append({
+                "hash": r["name_hash"],
+                "name": r["name"],
+                "price": float(r["price"]) if r.get("price") is not None else None,
+                "qty": float(r["qty"]) if r.get("qty") is not None else None,
+                "sum": float(r["sum"]) if r.get("sum") is not None else None,
+                "date_start": r["date_start"],
+            })
+
+        # Get elements_json
+        ej = spec_current.get_elements_json(s["document_id"])
+        if not ej:
+            continue
+
+        if isinstance(ej, dict) and "Value" in ej:
+            ej = ej["Value"]
+        if isinstance(ej, str):
+            elements = json.loads(ej)
+        elif isinstance(ej, list):
+            elements = ej
+        else:
+            elements = []
+
+        elements = [{k.lower(): v for k, v in el.items()} for el in elements]
+        doc_text = _elements_to_text(elements)
+
+        sse_send({"type": "extract_start", "supplement_id": sid, "filename": s["filename"]})
+
+        # LLM call in thread with keepalive
+        t1 = time.time()
+        done = threading.Event()
+        result = [None]
+        error = [None]
+
+        def do_llm():
+            try:
+                result[0] = call_llm(current_spec, doc_text, build_prompt_fn)
+            except Exception as e:
+                error[0] = str(e)
+            finally:
+                done.set()
+
+        t = threading.Thread(target=do_llm)
+        t.start()
+
+        while not done.wait(15):
+            sse_send({"type": "keepalive"})
+
+        t.join()
+        elapsed = round(time.time() - t1, 1)
+
+        if error[0]:
+            sse_send({"type": "extract_error", "supplement_id": sid,
+                       "filename": s["filename"], "error": error[0], "time_s": elapsed})
+            continue
+
+        llm_result, prompt_id = result[0]
+        ops = llm_result.get("ops", [])
+        mode = llm_result.get("mode", "partial")
+
+        # Enrich ops
+        id_map = {f"r{i+1}": r["hash"] for i, r in enumerate(current_spec)}
+        spec_names = {r["hash"]: r["name"] for r in current_spec}
+        for op in ops:
+            tid = op.get("target_id")
+            if tid and tid in id_map:
+                op["target_hash"] = id_map[tid]
+            th = op.get("target_hash")
+            if th and th in spec_names and not op.get("new_row", {}).get("name"):
+                op.setdefault("new_row", {})["name"] = spec_names[th]
+
+        sse_send({"type": "llm_done", "supplement_id": sid, "filename": s["filename"],
+                   "ops_count": len(ops), "mode": mode, "time_s": elapsed})
+
+        # Apply events
+        summary = spec_events.apply_ops(
+            contract_id, sid, s.get("document_id", ""), ops, prompt_id, llm_result
+        )
+        sse_send({"type": "applied", "supplement_id": sid, "summary": summary, "ops": ops})
+
+    total_time = round(time.time() - t0, 1)
+    sse_send({"type": "done", "total_time_s": total_time})
+
+
+def _elements_to_text(elements):
+    lines = []
+    for el in elements:
+        if el.get("type") == "paragraph":
+            prefix = f"[{el['style']}] " if el.get("style") else ""
+            lines.append(prefix + el.get("text", ""))
+        elif el.get("type") == "table":
+            rows = el.get("rows", [])
+            if rows:
+                ncols = len(rows[0])
+                lines.append(f"--- Таблица ({len(rows)}×{ncols}) ---")
+                for row in rows:
+                    cells = [str(c).replace("\n", " ").replace("|", "\\|") for c in row[:ncols]]
+                    lines.append("| " + " | ".join(cells) + " |")
+                lines.append("")
+    return "\n".join(lines)

deploy/services/unzip.py

@@ -0,0 +1,38 @@
+"""Unzip service — extract files from ZIP archive."""
+import zipfile, io, base64, json
+
+
+def handle_unzip(rfile, content_length):
+    """Parse ZIP upload, return list of extracted files."""
+    body = rfile.read(content_length)
+
+    # Find file data in raw multipart
+    # Simple approach: find ZIP bytes after header
+    idx = body.find(b"\r\n\r\n")
+    if idx < 0:
+        return {"ok": False, "error": "invalid multipart"}
+
+    # Find first boundary end
+    raw = body[idx + 4:]
+    # Find the actual ZIP data (after Content-Disposition etc)
+    zip_start = raw.find(b"PK\x03\x04")
+    if zip_start < 0:
+        return {"ok": False, "error": "not a ZIP file"}
+
+    zip_data = raw[zip_start:]
+
+    files = []
+    with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
+        for name in zf.namelist():
+            if name.endswith("/"):
+                continue
+            data = zf.read(name)
+            ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
+            files.append({
+                "filename": name,
+                "data_b64": base64.b64encode(data).decode(),
+                "ext": ext,
+                "size": len(data),
+            })
+
+    return {"ok": True, "files": files}

deploy/services/upload.py

@@ -0,0 +1,79 @@
+"""Upload service — accept file, store to DB, parse."""
+import uuid, base64, json
+from email.parser import BytesParser
+from db import documents, contracts, supplements
+from .parse import parse_file
+
+
+def handle_upload(rfile, content_type, content_length):
+    """Parse multipart upload, store in DB, return result dict."""
+    body = rfile.read(content_length)
+
+    msg = BytesParser().parsebytes(
+        b"Content-Type: " + content_type.encode() + b"\r\n\r\n" + body
+    )
+    filename = None
+    file_data = None
+    contract_id = ""
+
+    if msg.is_multipart():
+        for part in msg.get_payload():
+            fn = part.get_filename()
+            if fn:
+                filename = fn
+                file_data = part.get_payload(decode=True)
+            else:
+                disp = part.get("Content-Disposition", "")
+                if 'name="contract_id"' in disp:
+                    contract_id = part.get_payload(decode=True).decode("utf-8", errors="ignore").strip()
+
+    if not filename or not file_data:
+        return {"ok": False, "error": "no file in request"}
+
+    mime = _mime_for(filename)
+    b64 = base64.b64encode(file_data).decode()
+
+    # Удалить старый файл с тем же именем в этом контракте
+    if contract_id:
+        supplements.delete_by_document(contract_id, filename)
+
+    doc = documents.insert(filename, mime, b64)
+
+    # Создать контракт если нет
+    if not contract_id:
+        from datetime import datetime
+        now = datetime.now()
+        c = contracts.insert(f"б/н {now.strftime('%Y%m%d')}-{now.strftime('%H%M')}")
+        contract_id = c["id"]
+        supp_type = "initial"
+    else:
+        supp_type = "additional"
+
+    supplements.insert(contract_id, doc["id"], supp_type)
+
+    # Парсинг на VM
+    parsed = parse_file(filename, file_data)
+    if parsed and parsed.get("status") == "parsed":
+        documents.set_parsed(doc["id"], parsed["elements"])
+    elif parsed and parsed.get("status") == "error":
+        documents.set_error(doc["id"], parsed.get("error", "parse failed"))
+
+    return {
+        "ok": True,
+        "doc_id": doc["id"],
+        "contract_id": contract_id,
+        "filename": filename,
+        "size": len(file_data),
+        "parsed": parsed,
+    }
+
+
+def _mime_for(filename):
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    mime_map = {
+        "pdf": "application/pdf",
+        "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        "doc": "application/msword",
+        "zip": "application/zip",
+    }
+    return mime_map.get(ext, "application/octet-stream")