refactor: deploy-lucee + deploy-check — два независимых набора кода ВМ
This commit is contained in:
@@ -0,0 +1,257 @@
|
||||
"""
|
||||
Classify service — LLM-based document classification.
|
||||
|
||||
Архитектурное решение (Opus):
|
||||
- Отдельный сервис, не встроен в upload. Upload быстрый (0.5с), classify — медленный (2-10с/файл).
|
||||
- ThreadPoolExecutor(max_workers=4) — параллельная классификация с ограничением конкурентности,
|
||||
чтобы не положить api.aillm.ru при 2000 файлах.
|
||||
- Умная выжимка (_smart_extract): header ~1500 симв + regex-хиты по маркерам (договор/№/соглашение)
|
||||
из всего документа. Экономия токенов в 5-10 раз при сохранении точности.
|
||||
- Двухпроходная архитектура: LLM извлекает строки (тип/номер/дата/контрагент),
|
||||
Python в grouping.py нормализует и группирует детерминированно.
|
||||
"""
|
||||
import json, re, os
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
import httpx
|
||||
from db import documents as db_docs
|
||||
from llm_prompt import build_classify_prompt
|
||||
|
||||
log = __import__("logging").getLogger(__name__)
|
||||
|
||||
# Лимит одновременных запросов к LLM API
|
||||
# Увеличивать осторожно — api.aillm.ru может троттлить
|
||||
MAX_WORKERS = 4
|
||||
|
||||
LLM_URL = "https://api.aillm.ru/v1/chat/completions"
|
||||
LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "")
|
||||
LLM_MODEL = "gpt-oss-120b"
|
||||
|
||||
# ── Garbage filter (Stage 1: filename regex) ────────────────────
|
||||
_GARBAGE_FILENAME_RE = re.compile(
|
||||
r'(сч[её]т|акт|плат[её]ж|УПД|сверк|инвойс|invoice|payment|act)',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# ── Garbage filter (Stage 2: header keywords) ───────────────────
|
||||
_GARBAGE_HEADER_MARKERS = [
|
||||
'СЧЕТ-ФАКТУРА', 'СЧЕТ НА ОПЛАТУ', 'АКТ СВЕРКИ',
|
||||
'АКТ ОКАЗАННЫХ УСЛУГ', 'АКТ ВЫПОЛНЕННЫХ РАБОТ',
|
||||
'ПЛАТЁЖНОЕ ПОРУЧЕНИЕ', 'УНИВЕРСАЛЬНЫЙ ПЕРЕДАТОЧНЫЙ',
|
||||
'УПД', 'ПЛАТЕЖНОЕ ПОРУЧЕНИЕ',
|
||||
]
|
||||
|
||||
|
||||
def _is_garbage_by_filename(filename: str) -> bool:
|
||||
"""Stage 1: regex по имени файла — быстро, 0 токенов."""
|
||||
return bool(_GARBAGE_FILENAME_RE.search(filename))
|
||||
|
||||
|
||||
def _is_garbage_by_header(text: str) -> bool:
|
||||
"""Stage 2: ключевые слова в первых 2KB текста — быстро, 0 токенов."""
|
||||
header = text[:2000].upper()
|
||||
return any(marker in header for marker in _GARBAGE_HEADER_MARKERS)
|
||||
|
||||
|
||||
def _call_llm_classify(header_text):
|
||||
"""
|
||||
Прямой вызов LLM для классификации ОДНОГО документа.
|
||||
Возвращает (parsed_dict, raw_text, needed_fix).
|
||||
"""
|
||||
prompt, _ = build_classify_prompt(header_text)
|
||||
payload = {
|
||||
"model": LLM_MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": 1000,
|
||||
"temperature": 0.1,
|
||||
}
|
||||
with httpx.Client(http2=True, timeout=60) as client:
|
||||
resp = client.post(
|
||||
LLM_URL, json=payload,
|
||||
headers={"Authorization": f"Bearer {LLM_KEY}", "Content-Type": "application/json"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
raw = data.get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
parsed, needed_fix = _safe_json_parse(raw)
|
||||
return parsed, raw, needed_fix
|
||||
|
||||
|
||||
def classify_batch(batch_id):
|
||||
"""
|
||||
Классифицировать все документы в batch.
|
||||
Сбрасывает статус на 'pending' для всех перед началом.
|
||||
Параллельно (ThreadPoolExecutor) обрабатывает до MAX_WORKERS документов.
|
||||
Возвращает {ok, total, done, failed}.
|
||||
"""
|
||||
# Сбросить статус — allow re-classify after file changes
|
||||
db_docs.reset_classify_status(batch_id)
|
||||
pending = db_docs.list_pending(batch_id)
|
||||
if not pending:
|
||||
return {"ok": False, "error": "no pending documents"}
|
||||
|
||||
total = len(pending)
|
||||
done = 0
|
||||
failed = 0
|
||||
garbage = 0
|
||||
json_fixes = 0
|
||||
json_total = 0
|
||||
|
||||
def _classify_one(doc):
|
||||
"""Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
|
||||
nonlocal garbage, json_fixes, json_total
|
||||
try:
|
||||
# Stage 1: garbage by filename (0 tokens)
|
||||
if _is_garbage_by_filename(doc["filename"]):
|
||||
db_docs.set_classify_garbage(doc["id"], "filename_regex")
|
||||
garbage += 1
|
||||
return True
|
||||
|
||||
# Stage 2: garbage by header keywords (0 tokens)
|
||||
text = _smart_extract(doc["elements_json"])
|
||||
if _is_garbage_by_header(text):
|
||||
db_docs.set_classify_garbage(doc["id"], "header_keywords")
|
||||
garbage += 1
|
||||
return True
|
||||
|
||||
# Stage 3: LLM classification (only for remaining)
|
||||
db_docs.set_classify_processing(doc["id"]) # crash recovery marker
|
||||
result, raw, needed_fix = _call_llm_classify(text)
|
||||
json_total += 1
|
||||
if needed_fix:
|
||||
json_fixes += 1
|
||||
db_docs.set_classification(
|
||||
doc["id"],
|
||||
result.get("doc_type", "other"),
|
||||
result.get("own_number"),
|
||||
result.get("parent_number"),
|
||||
result.get("doc_date"),
|
||||
result.get("counterparty"),
|
||||
classify_raw=raw,
|
||||
classify_input=text,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
db_docs.set_classify_failed(doc["id"], str(e))
|
||||
return False
|
||||
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
|
||||
futures = {pool.submit(_classify_one, d): d for d in pending}
|
||||
for f in as_completed(futures):
|
||||
if f.result():
|
||||
done += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
return {"ok": True, "total": total, "done": done, "failed": failed,
|
||||
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3)}
|
||||
|
||||
|
||||
def _safe_json_parse(raw):
|
||||
"""Parse LLM response, fixing common JSON errors.
|
||||
Returns (parsed_dict, needed_fix: bool)."""
|
||||
if not raw:
|
||||
raise ValueError("empty LLM response")
|
||||
|
||||
text = raw.strip()
|
||||
# Strip markdown
|
||||
if "```json" in text:
|
||||
text = text.split("```json")[1].split("```")[0].strip()
|
||||
elif "```" in text:
|
||||
text = text.split("```")[1].split("```")[0].strip()
|
||||
|
||||
# Remove non-JSON prefix/suffix (LLM chatter)
|
||||
brace_start = text.find("{")
|
||||
brace_end = text.rfind("}")
|
||||
if brace_start >= 0 and brace_end > brace_start:
|
||||
text = text[brace_start:brace_end + 1]
|
||||
|
||||
# Try strict parse
|
||||
try:
|
||||
return json.loads(text), False
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
import re as _re
|
||||
# Collapse multiline
|
||||
text = _re.sub(r"\n\s*", " ", text)
|
||||
# Remove trailing commas
|
||||
text = _re.sub(r",\s*}", "}", text)
|
||||
text = _re.sub(r",\s*]", "]", text)
|
||||
|
||||
# Try again
|
||||
try:
|
||||
return json.loads(text), True
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Aggressive: try adding missing closing quotes/braces
|
||||
text = text.rstrip()
|
||||
if not text.endswith("}"):
|
||||
# Count unclosed quotes
|
||||
in_string = False
|
||||
for i, ch in enumerate(text):
|
||||
if ch == '"' and (i == 0 or text[i-1] != "\\"):
|
||||
in_string = not in_string
|
||||
if in_string:
|
||||
text += '"'
|
||||
text += "}"
|
||||
|
||||
return json.loads(text), True
|
||||
|
||||
|
||||
def _smart_extract(elements_json):
|
||||
"""
|
||||
Умная выжимка текста для классификации (решение Q3 от Opus).
|
||||
|
||||
Вместо отправки всего документа (дорого) или только header (теряет зарытые номера),
|
||||
используется гибрид:
|
||||
1. Первые ~1500 симв (титул, преамбула, стороны)
|
||||
2. Regex-хиты по маркерам «договор|№|соглашение|приложение|спецификация»
|
||||
из ВСЕГО документа
|
||||
3. Дедупликация, лимит 10 строк, склейка → ~3000 симв на вход LLM
|
||||
|
||||
Это покрывает и титульную зону, и зарытые ссылки в середине документа.
|
||||
"""
|
||||
if not elements_json:
|
||||
return ""
|
||||
|
||||
if isinstance(elements_json, str):
|
||||
try:
|
||||
elements = json.loads(elements_json)
|
||||
except json.JSONDecodeError:
|
||||
return elements_json[:2000]
|
||||
elif isinstance(elements_json, list):
|
||||
elements = elements_json
|
||||
else:
|
||||
return str(elements_json)[:2000]
|
||||
|
||||
# Build full text
|
||||
lines = []
|
||||
for el in elements:
|
||||
if isinstance(el, dict):
|
||||
t = el.get("type") or el.get("TYPE", "")
|
||||
if t == "paragraph":
|
||||
txt = el.get("text") or el.get("TEXT", "")
|
||||
if txt:
|
||||
lines.append(txt)
|
||||
elif t == "table":
|
||||
rows = el.get("rows") or el.get("ROWS", [])
|
||||
for row in rows:
|
||||
lines.append(" | ".join(str(c) for c in row))
|
||||
|
||||
full_text = "\n".join(lines)
|
||||
|
||||
# Header: first ~1500 chars
|
||||
header = full_text[:1500]
|
||||
|
||||
# Marker lines: grep for key patterns
|
||||
markers = re.findall(
|
||||
r'.{0,200}(?:договор|№|соглашен|приложен|специф|контрагент|заказчик|арендатор).{0,200}',
|
||||
full_text, re.IGNORECASE,
|
||||
)
|
||||
unique_markers = list(dict.fromkeys(markers))[:10]
|
||||
|
||||
combined = header + "\n---\n" + "\n".join(unique_markers)
|
||||
return combined[:3000]
|
||||
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Grouping service — match classified documents into contract groups.
|
||||
|
||||
Архитектурное решение (Opus, Q4 + Q6):
|
||||
- Двухпроходный гибрид: LLM извлекает строки (classify.py), Python нормализует и группирует.
|
||||
- Нормализация номеров: uppercase + только буквы/цифры.
|
||||
"МЭС-123-2024" == "МЭС 123/2024" после нормализации.
|
||||
- Группировка на бэкенде (не на фронте): Python regex/unicode надёжнее JS.
|
||||
- apply_groups(): создаёт contracts + supplements с авто-порядком по дате.
|
||||
"""
|
||||
import re
|
||||
from db import documents as db_docs
|
||||
from db import contracts as db_contracts
|
||||
from db import supplements as db_supplements
|
||||
|
||||
|
||||
def normalize_number(num):
|
||||
"""
|
||||
Нормализация номера договора для сравнения.
|
||||
Убирает всё кроме букв и цифр, приводит к uppercase.
|
||||
Пример: "МЭС-123-2024" → "МЭС1232024", "МЭС 123/2024" → "МЭС1232024".
|
||||
"""
|
||||
if not num:
|
||||
return ""
|
||||
return re.sub(r"[^A-Z0-9А-Я]", "", num.upper())
|
||||
|
||||
|
||||
def group_documents(batch_id):
|
||||
"""
|
||||
Сгруппировать классифицированные документы по контрактам.
|
||||
|
||||
Алгоритм:
|
||||
1. Отделить contract от supplement/specification
|
||||
2. Каждый contract → якорь группы
|
||||
3. Для каждого supplement: найти contract по parent_number (нормализованный)
|
||||
4. Оставшиеся supplement/spec → виртуальные группы по parent_number/own_number
|
||||
5. Совсем без номеров → группа "__unresolved__"
|
||||
6. Внутри группы сортировка по doc_date
|
||||
|
||||
Возвращает {ok, groups: [{contract_number, counterparty, documents: [...]}], total_docs}.
|
||||
"""
|
||||
docs = db_docs.list_by_batch(batch_id)
|
||||
classified = [d for d in docs if d.get("classify_status") == "classified"]
|
||||
|
||||
# Separate contracts and supplements
|
||||
contracts_list = []
|
||||
supplements_list = []
|
||||
|
||||
for d in classified:
|
||||
if d.get("doc_type") == "contract":
|
||||
contracts_list.append(d)
|
||||
else:
|
||||
supplements_list.append(d)
|
||||
|
||||
groups = []
|
||||
|
||||
# Each contract becomes a group
|
||||
for c in contracts_list:
|
||||
group = {
|
||||
"contract_number": c.get("own_number") or c.get("filename", ""),
|
||||
"counterparty": c.get("counterparty") or "",
|
||||
"documents": [c],
|
||||
}
|
||||
# Find supplements matching this contract
|
||||
c_norm = normalize_number(c.get("own_number"))
|
||||
for s in supplements_list:
|
||||
parent = normalize_number(s.get("parent_number") or "")
|
||||
own = normalize_number(s.get("own_number") or "")
|
||||
if parent == c_norm or own == c_norm:
|
||||
if s not in group["documents"]:
|
||||
group["documents"].append(s)
|
||||
|
||||
# Sort by date
|
||||
group["documents"].sort(key=lambda x: x.get("doc_date") or "")
|
||||
|
||||
# Remove matched supplements from the pool
|
||||
for s in group["documents"]:
|
||||
if s in supplements_list:
|
||||
supplements_list.remove(s)
|
||||
|
||||
groups.append(group)
|
||||
|
||||
# ── Virtual groups: unmatched supplements grouped by number ──────────
|
||||
# Group remaining supplements/specs by normalized parent_number (priority) or own_number
|
||||
virtual = {}
|
||||
for s in supplements_list:
|
||||
num = normalize_number(s.get("parent_number") or s.get("own_number") or "")
|
||||
if not num:
|
||||
continue # no number → stays in supplements_list for unresolved
|
||||
if num not in virtual:
|
||||
# Use counterparty from first doc in group
|
||||
cp = s.get("counterparty") or ""
|
||||
virtual[num] = {
|
||||
"contract_number": s.get("parent_number") or s.get("own_number") or "?",
|
||||
"counterparty": cp,
|
||||
"documents": [],
|
||||
}
|
||||
virtual[num]["documents"].append(s)
|
||||
# Update counterparty if current doc has a better one
|
||||
if not virtual[num]["counterparty"] and s.get("counterparty"):
|
||||
virtual[num]["counterparty"] = s.get("counterparty")
|
||||
|
||||
for vnum, vgroup in virtual.items():
|
||||
vgroup["documents"].sort(key=lambda x: x.get("doc_date") or "")
|
||||
groups.append(vgroup)
|
||||
# Remove grouped docs from supplements_list
|
||||
for s in vgroup["documents"]:
|
||||
supplements_list.remove(s)
|
||||
|
||||
# ── Unresolved: everything left (no number, failed, pending, other) ──
|
||||
unmatched = [s for s in supplements_list] # remaining after virtual grouping
|
||||
unmatched += [d for d in docs if d.get("classify_status") != "classified"]
|
||||
|
||||
if unmatched:
|
||||
groups.append({
|
||||
"contract_number": "__unresolved__",
|
||||
"counterparty": "",
|
||||
"documents": unmatched,
|
||||
})
|
||||
|
||||
return {"ok": True, "groups": groups, "total_docs": len(docs)}
|
||||
|
||||
|
||||
def apply_groups(batch_id, groups_data):
|
||||
"""
|
||||
Применить подтверждённые группы: создать contracts + supplements.
|
||||
|
||||
Вызывается из POST /api/apply-groups.
|
||||
Для каждой группы (кроме __unresolved__):
|
||||
1. Создать запись в contracts (number, client)
|
||||
2. Для каждого документа создать supplement (type='initial' для первого, 'additional' для остальных)
|
||||
3. Порядок supplements соответствует порядку документов в группе (сортировка по дате уже сделана)
|
||||
|
||||
Возвращает {ok, created: количество созданных supplements}.
|
||||
"""
|
||||
created = 0
|
||||
contract_ids = []
|
||||
for g in groups_data:
|
||||
contract_number = g.get("contract_number", "")
|
||||
if contract_number == "__unresolved__":
|
||||
continue
|
||||
counterparty = g.get("counterparty", "")
|
||||
docs = g.get("documents", [])
|
||||
|
||||
try:
|
||||
c = db_contracts.insert(contract_number, counterparty)
|
||||
contract_id = c["id"]
|
||||
contract_ids.append(contract_id)
|
||||
|
||||
for i, d in enumerate(docs):
|
||||
doc_id = d.get("id")
|
||||
if not doc_id:
|
||||
continue
|
||||
supp_type = "initial" if i == 0 else "additional"
|
||||
db_supplements.insert(contract_id, doc_id, supp_type)
|
||||
created += 1
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": f"apply_groups failed at {contract_number}: {e}"}
|
||||
|
||||
return {"ok": True, "created": created, "contract_ids": contract_ids}
|
||||
@@ -0,0 +1,37 @@
|
||||
"""LLM service — call LLM API."""
|
||||
import os, json
|
||||
import httpx
|
||||
|
||||
LLM_URL = "https://api.aillm.ru/v1/chat/completions"
|
||||
LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "")
|
||||
LLM_MODEL = "gpt-oss-120b"
|
||||
|
||||
|
||||
def call_llm(current_spec, doc_text, build_prompt_fn):
|
||||
"""Call LLM. Returns (parsed_result, prompt_id)."""
|
||||
prompt, prompt_id = build_prompt_fn(current_spec, doc_text)
|
||||
payload = {
|
||||
"model": LLM_MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": 8000,
|
||||
"temperature": 0.1,
|
||||
}
|
||||
with httpx.Client(http2=True, timeout=120, verify=True) as client:
|
||||
resp = client.post(
|
||||
LLM_URL,
|
||||
json=payload,
|
||||
headers={
|
||||
"Authorization": f"Bearer {LLM_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
raw_text = data.get("choices", [{}])[0].get("message", {}).get("content", "")
|
||||
json_text = raw_text
|
||||
if "```json" in json_text:
|
||||
json_text = json_text.split("```json")[1].split("```")[0]
|
||||
elif "```" in json_text:
|
||||
json_text = json_text.split("```")[1].split("```")[0]
|
||||
return json.loads(json_text.strip()), prompt_id
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Metrics — quality signals without golden dataset.
|
||||
|
||||
Checks that work immediately:
|
||||
- Arithmetic: sum == price * qty (free signal of LLM/data errors)
|
||||
- JSON fix rate: how often _safe_json_parse has to repair LLM output
|
||||
"""
|
||||
from decimal import Decimal, InvalidOperation
|
||||
|
||||
|
||||
def check_arithmetic(ops: list) -> list[dict]:
|
||||
"""Check sum == price * qty for ADD/UPDATE operations.
|
||||
Returns list of mismatches: [{action, name, price, qty, expected_sum, actual_sum, diff}]
|
||||
"""
|
||||
mismatches = []
|
||||
for op in ops:
|
||||
action = op.get("action", "")
|
||||
if action not in ("ADD", "UPDATE"):
|
||||
continue
|
||||
|
||||
row = op.get("new_row") or op.get("new_values") or {}
|
||||
price = _to_decimal(row.get("price"))
|
||||
qty = _to_decimal(row.get("qty"))
|
||||
actual_sum = _to_decimal(row.get("sum"))
|
||||
|
||||
if price is None or qty is None or actual_sum is None:
|
||||
continue # can't check without all three
|
||||
|
||||
expected = price * qty
|
||||
if expected != actual_sum:
|
||||
mismatches.append({
|
||||
"action": action,
|
||||
"name": row.get("name", "")[:100],
|
||||
"price": float(price),
|
||||
"qty": float(qty),
|
||||
"expected_sum": float(expected),
|
||||
"actual_sum": float(actual_sum),
|
||||
"diff": float(actual_sum - expected),
|
||||
})
|
||||
return mismatches
|
||||
|
||||
|
||||
class ClassifyMetrics:
|
||||
"""Track _safe_json_parse fix rate per batch."""
|
||||
def __init__(self):
|
||||
self.total = 0
|
||||
self.fixes = 0 # how many times JSON needed repair
|
||||
|
||||
def record(self, needed_fix: bool):
|
||||
self.total += 1
|
||||
if needed_fix:
|
||||
self.fixes += 1
|
||||
|
||||
@property
|
||||
def fix_rate(self) -> float:
|
||||
return self.fixes / self.total if self.total else 0.0
|
||||
|
||||
def summary(self) -> dict:
|
||||
return {
|
||||
"total_classifications": self.total,
|
||||
"json_fixes": self.fixes,
|
||||
"json_fix_rate": round(self.fix_rate, 3),
|
||||
}
|
||||
|
||||
|
||||
def _to_decimal(val) -> Decimal | None:
|
||||
"""Safe conversion to Decimal."""
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
return Decimal(str(val))
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Parse service — PDF (pdfplumber), DOCX (python-docx), plain text fallback."""
|
||||
import io
|
||||
|
||||
|
||||
def parse_file(filename, data):
|
||||
"""Parse file bytes → {status, elements, element_count}."""
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
try:
|
||||
if ext == "pdf":
|
||||
return _parse_pdf(data)
|
||||
elif ext == "docx":
|
||||
return _parse_docx(data)
|
||||
elif ext == "doc":
|
||||
return _parse_docx(data) # python-docx handles most .doc
|
||||
else:
|
||||
return _parse_text(data)
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": str(e), "element_count": 0}
|
||||
|
||||
|
||||
def _parse_pdf(data):
|
||||
"""Parse PDF with pdfplumber — preserves table structure (unlike PyPDF2)."""
|
||||
import pdfplumber
|
||||
elements = []
|
||||
with pdfplumber.open(io.BytesIO(data)) as pdf:
|
||||
for page in pdf.pages:
|
||||
# Tables first — preserves column structure critical for specs
|
||||
tables = page.extract_tables()
|
||||
for table in tables:
|
||||
if table:
|
||||
rows = []
|
||||
for row in table:
|
||||
if row:
|
||||
cells = [str(cell or "").strip() for cell in row]
|
||||
if any(cells):
|
||||
rows.append(cells)
|
||||
if rows:
|
||||
elements.append({"type": "table", "rows": rows})
|
||||
# Remaining text as paragraphs
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
for line in text.split("\n"):
|
||||
line = line.strip()
|
||||
if line:
|
||||
elements.append({"type": "paragraph", "text": line, "style": ""})
|
||||
return {"status": "parsed", "element_count": len(elements), "elements": elements}
|
||||
|
||||
|
||||
def _parse_docx(data):
|
||||
import docx
|
||||
doc = docx.Document(io.BytesIO(data))
|
||||
elements = []
|
||||
|
||||
for block in doc.element.body:
|
||||
tag = block.tag.split("}")[-1] if "}" in block.tag else block.tag
|
||||
if tag == "p":
|
||||
text = _extract_paragraph_text(block)
|
||||
if text:
|
||||
elements.append({"type": "paragraph", "text": text, "style": ""})
|
||||
elif tag == "tbl":
|
||||
rows = []
|
||||
for tr in block.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tr"):
|
||||
cells = []
|
||||
for tc in tr.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tc"):
|
||||
cell_text = "".join(t.text or "" for t in tc.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t"))
|
||||
cells.append(cell_text.strip())
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
if rows:
|
||||
elements.append({"type": "table", "rows": rows})
|
||||
|
||||
return {"status": "parsed", "element_count": len(elements), "elements": elements}
|
||||
|
||||
|
||||
def _extract_paragraph_text(p_elem):
|
||||
texts = []
|
||||
for t in p_elem.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t"):
|
||||
if t.text:
|
||||
texts.append(t.text)
|
||||
return "".join(texts).strip()
|
||||
|
||||
|
||||
def _parse_text(data):
|
||||
text = data.decode("utf-8", errors="ignore")[:5000]
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
return {"status": "parsed", "element_count": len(lines),
|
||||
"elements": [{"type": "paragraph", "text": l, "style": ""} for l in lines]}
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Process service — SSE pipeline: reset → supplements → LLM → apply."""
|
||||
import json, time, threading
|
||||
from db import supplements, spec_current, spec_events
|
||||
from .metrics import check_arithmetic
|
||||
|
||||
|
||||
def run_pipeline(contract_id, order_ids, sse_send, build_prompt_fn):
|
||||
"""Run full SSE comparison pipeline. sse_send is a callback(dict)."""
|
||||
t0 = time.time()
|
||||
|
||||
# 0. Reset
|
||||
spec_events.reset(contract_id)
|
||||
|
||||
# 1. Get supplements with parsed documents
|
||||
supps = supplements.list_by_contract(contract_id)
|
||||
if order_ids:
|
||||
order_list = [x.strip() for x in order_ids.split(",") if x.strip()]
|
||||
order_map = {oid: i for i, oid in enumerate(order_list)}
|
||||
supps.sort(key=lambda s: order_map.get(s["id"], 999999))
|
||||
|
||||
if not supps:
|
||||
sse_send({"type": "error", "message": "Нет распарсенных файлов"})
|
||||
return
|
||||
|
||||
from .llm import call_llm
|
||||
|
||||
for s in supps:
|
||||
sid = s["id"]
|
||||
|
||||
# Current spec
|
||||
cur = spec_current.list_by_contract(contract_id)
|
||||
current_spec = []
|
||||
for r in cur:
|
||||
current_spec.append({
|
||||
"hash": r["name_hash"],
|
||||
"name": r["name"],
|
||||
"price": float(r["price"]) if r.get("price") is not None else None,
|
||||
"qty": float(r["qty"]) if r.get("qty") is not None else None,
|
||||
"sum": float(r["sum"]) if r.get("sum") is not None else None,
|
||||
"date_start": r["date_start"],
|
||||
})
|
||||
|
||||
# Get elements_json
|
||||
ej = spec_current.get_elements_json(s["document_id"])
|
||||
if not ej:
|
||||
continue
|
||||
|
||||
if isinstance(ej, dict) and "Value" in ej:
|
||||
ej = ej["Value"]
|
||||
if isinstance(ej, str):
|
||||
elements = json.loads(ej)
|
||||
elif isinstance(ej, list):
|
||||
elements = ej
|
||||
else:
|
||||
elements = []
|
||||
|
||||
elements = [{k.lower(): v for k, v in el.items()} for el in elements]
|
||||
doc_text = _elements_to_text(elements)
|
||||
|
||||
sse_send({"type": "extract_start", "supplement_id": sid, "filename": s["filename"]})
|
||||
|
||||
# LLM call in thread with keepalive
|
||||
t1 = time.time()
|
||||
done = threading.Event()
|
||||
result = [None]
|
||||
error = [None]
|
||||
|
||||
def do_llm():
|
||||
try:
|
||||
result[0] = call_llm(current_spec, doc_text, build_prompt_fn)
|
||||
except Exception as e:
|
||||
error[0] = str(e)
|
||||
finally:
|
||||
done.set()
|
||||
|
||||
t = threading.Thread(target=do_llm)
|
||||
t.start()
|
||||
|
||||
while not done.wait(15):
|
||||
sse_send({"type": "keepalive"})
|
||||
|
||||
t.join()
|
||||
elapsed = round(time.time() - t1, 1)
|
||||
|
||||
if error[0]:
|
||||
sse_send({"type": "extract_error", "supplement_id": sid,
|
||||
"filename": s["filename"], "error": error[0], "time_s": elapsed})
|
||||
continue
|
||||
|
||||
llm_result, prompt_id = result[0]
|
||||
ops = llm_result.get("ops", [])
|
||||
mode = llm_result.get("mode", "partial")
|
||||
|
||||
# Enrich ops
|
||||
id_map = {f"r{i+1}": r["hash"] for i, r in enumerate(current_spec)}
|
||||
spec_names = {r["hash"]: r["name"] for r in current_spec}
|
||||
for op in ops:
|
||||
tid = op.get("target_id")
|
||||
if tid and tid in id_map:
|
||||
op["target_hash"] = id_map[tid]
|
||||
th = op.get("target_hash")
|
||||
if th and th in spec_names and not op.get("new_row", {}).get("name"):
|
||||
op.setdefault("new_row", {})["name"] = spec_names[th]
|
||||
|
||||
sse_send({"type": "llm_done", "supplement_id": sid, "filename": s["filename"],
|
||||
"ops_count": len(ops), "mode": mode, "time_s": elapsed})
|
||||
|
||||
# Apply events
|
||||
summary = spec_events.apply_ops(
|
||||
contract_id, sid, s.get("document_id", ""), ops, prompt_id, llm_result
|
||||
)
|
||||
# Arithmetic quality check (free signal, no golden dataset needed)
|
||||
arith_mismatches = check_arithmetic(ops)
|
||||
if arith_mismatches:
|
||||
summary["arithmetic_mismatches"] = len(arith_mismatches)
|
||||
sse_send({"type": "applied", "supplement_id": sid, "summary": summary, "ops": ops})
|
||||
|
||||
total_time = round(time.time() - t0, 1)
|
||||
sse_send({"type": "done", "total_time_s": total_time})
|
||||
|
||||
|
||||
def _elements_to_text(elements):
|
||||
lines = []
|
||||
for el in elements:
|
||||
if el.get("type") == "paragraph":
|
||||
prefix = f"[{el['style']}] " if el.get("style") else ""
|
||||
lines.append(prefix + el.get("text", ""))
|
||||
elif el.get("type") == "table":
|
||||
rows = el.get("rows", [])
|
||||
if rows:
|
||||
ncols = len(rows[0])
|
||||
lines.append(f"--- Таблица ({len(rows)}×{ncols}) ---")
|
||||
for row in rows:
|
||||
cells = [str(c).replace("\n", " ").replace("|", "\\|") for c in row[:ncols]]
|
||||
lines.append("| " + " | ".join(cells) + " |")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Unzip service — extract files from ZIP archive (no DB, no parse)."""
|
||||
import zipfile, io, base64
|
||||
|
||||
|
||||
def handle_unzip(rfile, content_length):
|
||||
"""Parse ZIP upload, return list of extracted files as base64."""
|
||||
body = rfile.read(content_length)
|
||||
|
||||
# Find file data in raw multipart
|
||||
idx = body.find(b"\r\n\r\n")
|
||||
if idx < 0:
|
||||
return {"ok": False, "error": "invalid multipart"}
|
||||
|
||||
raw = body[idx + 4:]
|
||||
zip_start = raw.find(b"PK\x03\x04")
|
||||
if zip_start < 0:
|
||||
return {"ok": False, "error": "not a ZIP file"}
|
||||
|
||||
zip_data = raw[zip_start:]
|
||||
|
||||
files = []
|
||||
with zipfile.ZipFile(io.BytesIO(zip_data)) as zf:
|
||||
for name in zf.namelist():
|
||||
if name.endswith("/"):
|
||||
continue
|
||||
data = zf.read(name)
|
||||
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
|
||||
files.append({
|
||||
"filename": name,
|
||||
"data_b64": base64.b64encode(data).decode(),
|
||||
"ext": ext,
|
||||
"size": len(data),
|
||||
})
|
||||
|
||||
return {"ok": True, "files": files}
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Upload service — accept file, store to DB, parse."""
|
||||
import uuid, base64, json, io, cgi, os
|
||||
from db import documents, contracts, supplements
|
||||
from .parse import parse_file
|
||||
|
||||
MAX_FILE_SIZE = 100 * 1024 * 1024 # 100 MB
|
||||
|
||||
|
||||
def handle_upload(rfile, content_type, content_length):
|
||||
"""Parse multipart upload, store in DB, return result dict."""
|
||||
# Validate content_length
|
||||
try:
|
||||
cl = int(content_length)
|
||||
except (TypeError, ValueError):
|
||||
return {"ok": False, "error": "invalid content-length"}
|
||||
if cl <= 0:
|
||||
return {"ok": False, "error": "empty request"}
|
||||
if cl > MAX_FILE_SIZE:
|
||||
return {"ok": False, "error": f"file too large (max {MAX_FILE_SIZE // 1024 // 1024}MB)"}
|
||||
|
||||
body = rfile.read(cl)
|
||||
|
||||
environ = {
|
||||
"REQUEST_METHOD": "POST",
|
||||
"CONTENT_TYPE": content_type,
|
||||
"CONTENT_LENGTH": str(content_length),
|
||||
}
|
||||
fs = cgi.FieldStorage(fp=io.BytesIO(body), environ=environ, keep_blank_values=True)
|
||||
|
||||
filename = None
|
||||
file_data = None
|
||||
contract_id = ""
|
||||
|
||||
if "files" in fs:
|
||||
item = fs["files"]
|
||||
if isinstance(item, list):
|
||||
item = item[0]
|
||||
filename = os.path.basename(item.filename) if item.filename else None
|
||||
if filename and (".." in filename or "/" in filename or "\\" in filename):
|
||||
return {"ok": False, "error": "invalid filename"}
|
||||
file_data = item.file.read() if hasattr(item, "file") else item.value
|
||||
if isinstance(file_data, str):
|
||||
file_data = file_data.encode("utf-8")
|
||||
|
||||
if "contract_id" in fs:
|
||||
contract_id = fs.getfirst("contract_id", "")
|
||||
# Validate UUID
|
||||
if contract_id:
|
||||
try:
|
||||
uuid.UUID(contract_id)
|
||||
except (ValueError, AttributeError):
|
||||
contract_id = ""
|
||||
|
||||
batch_id = fs.getfirst("batch_id", None) if "batch_id" in fs else None
|
||||
# Validate UUID
|
||||
if batch_id:
|
||||
try:
|
||||
uuid.UUID(batch_id)
|
||||
except (ValueError, AttributeError):
|
||||
batch_id = None
|
||||
|
||||
# zip_source — имя родительского ZIP-архива (для визуальной группировки)
|
||||
zip_source = None
|
||||
if "zip_source" in fs:
|
||||
raw_zip = fs.getfirst("zip_source", "")
|
||||
if raw_zip:
|
||||
zip_source = os.path.basename(raw_zip)[:255] # защита от path traversal + лимит
|
||||
|
||||
if not filename or not file_data:
|
||||
return {"ok": False, "error": "no file in request"}
|
||||
|
||||
mime = _mime_for(filename)
|
||||
b64 = base64.b64encode(file_data).decode()
|
||||
|
||||
if contract_id:
|
||||
try:
|
||||
supplements.delete_by_document(contract_id, filename)
|
||||
except Exception:
|
||||
pass # old record may not exist or FK issue — proceed with insert
|
||||
|
||||
doc = documents.insert(filename, mime, b64, batch_id=batch_id, zip_source=zip_source)
|
||||
|
||||
if not contract_id:
|
||||
from datetime import datetime
|
||||
now = datetime.now()
|
||||
c = contracts.insert(f"б/н {now.strftime('%Y%m%d')}-{now.strftime('%H%M')}")
|
||||
contract_id = c["id"]
|
||||
supp_type = "initial"
|
||||
else:
|
||||
supp_type = "additional"
|
||||
|
||||
supplements.insert(contract_id, doc["id"], supp_type)
|
||||
|
||||
parsed = parse_file(filename, file_data)
|
||||
if parsed and parsed.get("status") == "parsed":
|
||||
documents.set_parsed(doc["id"], parsed["elements"])
|
||||
elif parsed and parsed.get("status") == "error":
|
||||
documents.set_error(doc["id"], parsed.get("error", "parse failed"))
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"doc_id": doc["id"],
|
||||
"contract_id": contract_id,
|
||||
"filename": filename,
|
||||
"size": len(file_data),
|
||||
"parsed": parsed,
|
||||
}
|
||||
|
||||
|
||||
def _mime_for(filename):
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
mime_map = {
|
||||
"pdf": "application/pdf",
|
||||
"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"doc": "application/msword",
|
||||
"zip": "application/zip",
|
||||
}
|
||||
return mime_map.get(ext, "application/octet-stream")
|
||||
Reference in New Issue
Block a user