revert: app/ → site/ (drhider proves site/ works fine)
This commit is contained in:
@@ -0,0 +1,267 @@
|
||||
"""
|
||||
Classify service — LLM-based document classification.
|
||||
|
||||
Архитектурное решение (Opus):
|
||||
- Отдельный сервис, не встроен в upload. Upload быстрый (0.5с), classify — медленный (2-10с/файл).
|
||||
- ThreadPoolExecutor(max_workers=4) — параллельная классификация с ограничением конкурентности,
|
||||
чтобы не положить api.aillm.ru при 2000 файлах.
|
||||
- Умная выжимка (_smart_extract): header ~1500 симв + regex-хиты по маркерам (договор/№/соглашение)
|
||||
из всего документа. Экономия токенов в 5-10 раз при сохранении точности.
|
||||
- Двухпроходная архитектура: LLM извлекает строки (тип/номер/дата/контрагент),
|
||||
Python в grouping.py нормализует и группирует детерминированно.
|
||||
"""
|
||||
import json, re, os
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
import httpx
|
||||
from site.db import documents as db_docs
|
||||
from site.llm_prompt import build_classify_prompt
|
||||
|
||||
log = __import__("logging").getLogger(__name__)
|
||||
|
||||
# Лимит одновременных запросов к LLM API
|
||||
# Увеличивать осторожно — api.aillm.ru может троттлить
|
||||
MAX_WORKERS = 4
|
||||
|
||||
LLM_URL = "https://api.aillm.ru/v1/chat/completions"
|
||||
LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "")
|
||||
LLM_MODEL = "gpt-oss-120b"
|
||||
|
||||
# Ленивый singleton — обратная совместимость
|
||||
_classify_llm = None
|
||||
|
||||
|
||||
def _get_classify_client():
|
||||
global _classify_llm
|
||||
if _classify_llm is None:
|
||||
from site.services.llm_client import HttpxLLMClient
|
||||
_classify_llm = HttpxLLMClient(url=LLM_URL, key=LLM_KEY, model=LLM_MODEL, max_tokens=1000, timeout=60)
|
||||
return _classify_llm
|
||||
|
||||
# ── Garbage filter (Stage 1: filename regex) ────────────────────
|
||||
_GARBAGE_FILENAME_RE = re.compile(
|
||||
r'(сч[её]т|акт|плат[её]ж|УПД|сверк|инвойс|invoice|payment|act)',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# ── Garbage filter (Stage 2: header keywords) ───────────────────
|
||||
_GARBAGE_HEADER_MARKERS = [
|
||||
'СЧЕТ-ФАКТУРА', 'СЧЕТ НА ОПЛАТУ', 'АКТ СВЕРКИ',
|
||||
'АКТ ОКАЗАННЫХ УСЛУГ', 'АКТ ВЫПОЛНЕННЫХ РАБОТ',
|
||||
'ПЛАТЁЖНОЕ ПОРУЧЕНИЕ', 'УНИВЕРСАЛЬНЫЙ ПЕРЕДАТОЧНЫЙ',
|
||||
'УПД', 'ПЛАТЕЖНОЕ ПОРУЧЕНИЕ',
|
||||
]
|
||||
|
||||
|
||||
def _is_garbage_by_filename(filename: str) -> bool:
|
||||
"""Stage 1: regex по имени файла — быстро, 0 токенов."""
|
||||
return bool(_GARBAGE_FILENAME_RE.search(filename))
|
||||
|
||||
|
||||
def _is_garbage_by_header(text: str) -> bool:
|
||||
"""Stage 2: ключевые слова в первых 2KB текста — быстро, 0 токенов."""
|
||||
header = text[:2000].upper()
|
||||
return any(marker in header for marker in _GARBAGE_HEADER_MARKERS)
|
||||
|
||||
|
||||
def _call_llm_classify(header_text, llm_client=None):
|
||||
"""
|
||||
Прямой вызов LLM для классификации ОДНОГО документа.
|
||||
Возвращает (parsed_dict, raw_text, needed_fix).
|
||||
llm_client: LLMClient (optional). Default — HttpxLLMClient.
|
||||
"""
|
||||
if llm_client is None:
|
||||
llm_client = _get_classify_client()
|
||||
|
||||
prompt, _ = build_classify_prompt(header_text)
|
||||
raw_text = llm_client.complete(prompt)
|
||||
parsed, needed_fix = _safe_json_parse(raw_text)
|
||||
return parsed, raw_text, needed_fix
|
||||
|
||||
|
||||
def classify_batch(batch_id, llm_client=None, repo=None):
|
||||
"""
|
||||
Классифицировать все документы в batch.
|
||||
Сбрасывает статус на 'pending' для всех перед началом.
|
||||
Параллельно (ThreadPoolExecutor) обрабатывает до MAX_WORKERS документов.
|
||||
Возвращает {ok, total, done, failed, garbage, json_fix_rate}.
|
||||
llm_client: LLMClient (optional, default — HttpxLLMClient)
|
||||
repo: Repository (optional, default — direct db.* calls)
|
||||
"""
|
||||
_db = repo if repo else db_docs
|
||||
_llm = llm_client if llm_client else _get_classify_client()
|
||||
|
||||
# Сбросить статус — allow re-classify after file changes
|
||||
_db.reset_classify_status(batch_id)
|
||||
pending = _db.list_pending(batch_id)
|
||||
if not pending:
|
||||
return {"ok": False, "error": "no pending documents"}
|
||||
|
||||
total = len(pending)
|
||||
done = 0
|
||||
failed = 0
|
||||
garbage = 0
|
||||
json_fixes = 0
|
||||
json_total = 0
|
||||
type_counts = {} # doc_type → count for batch summary
|
||||
|
||||
def _classify_one(doc):
|
||||
"""Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
|
||||
nonlocal garbage, json_fixes, json_total, type_counts
|
||||
try:
|
||||
# Stage 1: garbage by filename (0 tokens)
|
||||
if _is_garbage_by_filename(doc["filename"]):
|
||||
_db.set_classify_garbage(doc["id"], "filename_regex")
|
||||
garbage += 1
|
||||
return True
|
||||
|
||||
# Stage 2: garbage by header keywords (0 tokens)
|
||||
text = _smart_extract(doc["elements_json"])
|
||||
if _is_garbage_by_header(text):
|
||||
_db.set_classify_garbage(doc["id"], "header_keywords")
|
||||
garbage += 1
|
||||
return True
|
||||
|
||||
# Stage 3: LLM classification (only for remaining)
|
||||
_db.set_classify_processing(doc["id"]) # crash recovery marker
|
||||
result, raw, needed_fix = _call_llm_classify(text, _llm)
|
||||
json_total += 1
|
||||
if needed_fix:
|
||||
json_fixes += 1
|
||||
dtype = result.get("doc_type", "other")
|
||||
type_counts[dtype] = type_counts.get(dtype, 0) + 1
|
||||
_db.set_classification(
|
||||
doc["id"],
|
||||
result.get("doc_type", "other"),
|
||||
result.get("own_number"),
|
||||
result.get("parent_number"),
|
||||
result.get("doc_date"),
|
||||
result.get("counterparty"),
|
||||
classify_raw=raw,
|
||||
classify_input=text,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
_db.set_classify_failed(doc["id"], str(e))
|
||||
return False
|
||||
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
|
||||
futures = {pool.submit(_classify_one, d): d for d in pending}
|
||||
for f in as_completed(futures):
|
||||
if f.result():
|
||||
done += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
return {"ok": True, "total": total, "done": done, "failed": failed,
|
||||
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3),
|
||||
"types": type_counts, "summary": f"{total} total, {done} classified, {failed} failed, {garbage} garbage"}
|
||||
|
||||
|
||||
def _safe_json_parse(raw):
|
||||
"""Parse LLM response, fixing common JSON errors.
|
||||
Returns (parsed_dict, needed_fix: bool)."""
|
||||
if not raw:
|
||||
raise ValueError("empty LLM response")
|
||||
|
||||
text = raw.strip()
|
||||
# Strip markdown
|
||||
if "```json" in text:
|
||||
text = text.split("```json")[1].split("```")[0].strip()
|
||||
elif "```" in text:
|
||||
text = text.split("```")[1].split("```")[0].strip()
|
||||
|
||||
# Remove non-JSON prefix/suffix (LLM chatter)
|
||||
brace_start = text.find("{")
|
||||
brace_end = text.rfind("}")
|
||||
if brace_start >= 0 and brace_end > brace_start:
|
||||
text = text[brace_start:brace_end + 1]
|
||||
|
||||
# Try strict parse
|
||||
try:
|
||||
return json.loads(text), False
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
import re as _re
|
||||
# Collapse multiline
|
||||
text = _re.sub(r"\n\s*", " ", text)
|
||||
# Remove trailing commas
|
||||
text = _re.sub(r",\s*}", "}", text)
|
||||
text = _re.sub(r",\s*]", "]", text)
|
||||
|
||||
# Try again
|
||||
try:
|
||||
return json.loads(text), True
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Aggressive: try adding missing closing quotes/braces
|
||||
text = text.rstrip()
|
||||
if not text.endswith("}"):
|
||||
# Count unclosed quotes
|
||||
in_string = False
|
||||
for i, ch in enumerate(text):
|
||||
if ch == '"' and (i == 0 or text[i-1] != "\\"):
|
||||
in_string = not in_string
|
||||
if in_string:
|
||||
text += '"'
|
||||
text += "}"
|
||||
|
||||
return json.loads(text), True
|
||||
|
||||
|
||||
def _smart_extract(elements_json):
|
||||
"""
|
||||
Умная выжимка текста для классификации (решение Q3 от Opus).
|
||||
|
||||
Вместо отправки всего документа (дорого) или только header (теряет зарытые номера),
|
||||
используется гибрид:
|
||||
1. Первые ~1500 симв (титул, преамбула, стороны)
|
||||
2. Regex-хиты по маркерам «договор|№|соглашение|приложение|спецификация»
|
||||
из ВСЕГО документа
|
||||
3. Дедупликация, лимит 10 строк, склейка → ~3000 симв на вход LLM
|
||||
|
||||
Это покрывает и титульную зону, и зарытые ссылки в середине документа.
|
||||
"""
|
||||
if not elements_json:
|
||||
return ""
|
||||
|
||||
if isinstance(elements_json, str):
|
||||
try:
|
||||
elements = json.loads(elements_json)
|
||||
except json.JSONDecodeError:
|
||||
return elements_json[:2000]
|
||||
elif isinstance(elements_json, list):
|
||||
elements = elements_json
|
||||
else:
|
||||
return str(elements_json)[:2000]
|
||||
|
||||
# Build full text
|
||||
lines = []
|
||||
for el in elements:
|
||||
if isinstance(el, dict):
|
||||
t = el.get("type") or el.get("TYPE", "")
|
||||
if t == "paragraph":
|
||||
txt = el.get("text") or el.get("TEXT", "")
|
||||
if txt:
|
||||
lines.append(txt)
|
||||
elif t == "table":
|
||||
rows = el.get("rows") or el.get("ROWS", [])
|
||||
for row in rows:
|
||||
lines.append(" | ".join(str(c) for c in row))
|
||||
|
||||
full_text = "\n".join(lines)
|
||||
|
||||
# Header: first ~1500 chars
|
||||
header = full_text[:1500]
|
||||
|
||||
# Marker lines: grep for key patterns
|
||||
markers = re.findall(
|
||||
r'.{0,200}(?:договор|№|соглашен|приложен|специф|контрагент|заказчик|арендатор).{0,200}',
|
||||
full_text, re.IGNORECASE,
|
||||
)
|
||||
unique_markers = list(dict.fromkeys(markers))[:10]
|
||||
|
||||
combined = header + "\n---\n" + "\n".join(unique_markers)
|
||||
return combined[:3000]
|
||||
@@ -0,0 +1,549 @@
|
||||
"""
|
||||
DrHider — обфускация документов (двухпроходная, в памяти, без БД).
|
||||
|
||||
Проход 1: собрать все сущности из всех файлов → глобальный словарь замен.
|
||||
Проход 2: применить замены → собрать ZIP с обфусцированными файлами + mapping.csv.
|
||||
|
||||
Согласованность: одна и та же сущность во всех файлах → одно и то же фиктивное значение.
|
||||
"""
|
||||
DRHIDER_VERSION = "1.3"
|
||||
import io
|
||||
import csv
|
||||
import re
|
||||
import random
|
||||
import string
|
||||
import zipfile
|
||||
import logging
|
||||
import os
|
||||
from typing import Dict, List, Tuple, Callable, Optional
|
||||
|
||||
log = logging.getLogger("drhider")
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# Regex-паттерны для обнаружения сущностей
|
||||
# ═══════════════════════════════════════════
|
||||
|
||||
ENTITY_PATTERNS: Dict[str, str] = {
|
||||
"phone": r'(?<!\d)(?:\+7|8)[\s\-]?\(?\d{3}\)?[\s\-]?\d{3}[\s\-]?\d{2}[\s\-]?\d{2}(?!\d)',
|
||||
"email": r'\b[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}\b',
|
||||
# 12-значный ИНН ДО 10-значного (иначе 12-значный матчится как 10)
|
||||
"inn_fl": r'ИНН\s*\d{12}',
|
||||
"inn_ul": r'ИНН\s*\d{10}',
|
||||
"ogrn": r'ОГРН\s*\d{13}',
|
||||
"kpp": r'КПП\s*\d{9}',
|
||||
"bik": r'БИК\s*\d{9}',
|
||||
"rs": r'(?:р/с|расч[её]тный\s*сч[её]т)\s*\d{20}',
|
||||
"ks": r'(?:к/с|кор/сч|корр?[еи]?спондентский\s*сч[её]т)\s*\d{20}',
|
||||
"passport": r'(?:паспорт|серия\s+номер)[\s:№]*\d{2}\s*\d{2}\s*\d{6,7}',
|
||||
}
|
||||
|
||||
# Фирмы — обнаружение по шаблону
|
||||
COMPANY_PATTERN = re.compile(
|
||||
r'(?:ООО|ЗАО|ОАО|АО|ПАО|ИП|ТОО)\s+(?:«[^»]+»|"[^"]+"|\u201c[^\u201d]+\u201d|[А-ЯA-Z][\w\-\.]{2,40})',
|
||||
re.IGNORECASE
|
||||
)
|
||||
|
||||
# ФИО — Фамилия + инициалы или полное имя (только кириллица)
|
||||
PERSON_PATTERN = re.compile(
|
||||
r'\b[А-Я][а-я]+\s+[А-Я]\.[А-Я]\.'
|
||||
r'|\b[А-Я][а-я]+\s+[А-Я][а-я]+\s+[А-Я][а-я]+'
|
||||
)
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# Генераторы фиктивных значений
|
||||
# ═══════════════════════════════════════════
|
||||
|
||||
# Словари русских имён
|
||||
RU_SURNAMES = ["Иванов", "Смирнов", "Кузнецов", "Попов", "Васильев", "Петров",
|
||||
"Соколов", "Михайлов", "Новиков", "Фёдоров", "Морозов", "Волков", "Алексеев",
|
||||
"Лебедев", "Семёнов", "Егоров", "Павлов", "Козлов", "Степанов", "Николаев"]
|
||||
RU_NAMES = ["Александр", "Дмитрий", "Сергей", "Андрей", "Алексей", "Максим",
|
||||
"Евгений", "Иван", "Михаил", "Николай", "Владимир", "Павел", "Виктор", "Олег"]
|
||||
RU_PATRONYMICS = ["Александрович", "Дмитриевич", "Сергеевич", "Андреевич",
|
||||
"Алексеевич", "Иванович", "Михайлович", "Николаевич", "Владимирович",
|
||||
"Павлович", "Викторович", "Олегович", "Евгеньевич", "Максимович"]
|
||||
|
||||
RU_CITIES = ["Москва", "Санкт-Петербург", "Новосибирск", "Екатеринбург",
|
||||
"Казань", "Нижний Новгород", "Челябинск", "Самара", "Омск", "Ростов-на-Дону",
|
||||
"Уфа", "Красноярск", "Воронеж", "Пермь", "Волгоград"]
|
||||
RU_STREETS = ["Ленина", "Мира", "Пушкина", "Гагарина", "Советская",
|
||||
"Кирова", "Октябрьская", "Молодёжная", "Садовая", "Центральная"]
|
||||
|
||||
FAKE_DOMAINS = ["example.ru", "mail.test", "company.local", "org.example.ru"]
|
||||
|
||||
|
||||
def _checksum_inn10(inn: str) -> str:
|
||||
"""Контрольная сумма для 10-значного ИНН."""
|
||||
coeffs = [2, 4, 10, 3, 5, 9, 4, 6, 8]
|
||||
s = sum(int(inn[i]) * coeffs[i] for i in range(9))
|
||||
return str((s % 11) % 10)
|
||||
|
||||
|
||||
def _checksum_inn12(inn: str) -> str:
|
||||
"""Контрольные суммы для 12-значного ИНН (первые 10 цифр)."""
|
||||
c1 = [7, 2, 4, 10, 3, 5, 9, 4, 6, 8]
|
||||
c2 = [3, 7, 2, 4, 10, 3, 5, 9, 4, 6, 8]
|
||||
s1 = sum(int(inn[i]) * c1[i] for i in range(10))
|
||||
n1 = (s1 % 11) % 10
|
||||
inn2 = inn + str(n1)
|
||||
s2 = sum(int(inn2[i]) * c2[i] for i in range(11))
|
||||
n2 = (s2 % 11) % 10
|
||||
return str(n1) + str(n2)
|
||||
|
||||
|
||||
def _checksum_ogrn(ogrn: str) -> str:
|
||||
"""Контрольная сумма для ОГРН (12 цифр → остаток от деления на 11)."""
|
||||
s = int(ogrn) % 11
|
||||
return str(s % 10)
|
||||
|
||||
|
||||
def _random_digits(n: int) -> str:
|
||||
return ''.join(random.choice(string.digits) for _ in range(n))
|
||||
|
||||
|
||||
def _random_letters(n: int) -> str:
|
||||
return ''.join(random.choice(string.ascii_lowercase) for _ in range(n))
|
||||
|
||||
|
||||
def generate_phone(_: str) -> str:
|
||||
code = random.choice(["495", "499", "812", "383", "343"])
|
||||
return f"+7 ({code}) {_random_digits(3)}-{_random_digits(2)}-{_random_digits(2)}"
|
||||
|
||||
|
||||
def generate_email(original: str) -> str:
|
||||
"""Генерирует email с тем же форматом."""
|
||||
domain = random.choice(FAKE_DOMAINS)
|
||||
local = _random_letters(random.randint(5, 10))
|
||||
return f"{local}@{domain}"
|
||||
|
||||
|
||||
def generate_inn10(original: str) -> str:
|
||||
prefix = re.match(r'ИНН\s*', original, re.IGNORECASE).group(0)
|
||||
base = f"{random.randint(1,9)}{_random_digits(8)}"
|
||||
return prefix + base + _checksum_inn10(base)
|
||||
|
||||
|
||||
def generate_inn12(original: str) -> str:
|
||||
prefix = re.match(r'ИНН\s*', original, re.IGNORECASE).group(0)
|
||||
base = f"{random.randint(1,9)}{_random_digits(9)}"
|
||||
return prefix + base + _checksum_inn12(base)
|
||||
|
||||
|
||||
def generate_ogrn(original: str) -> str:
|
||||
prefix = re.match(r'ОГРН\s*', original, re.IGNORECASE).group(0)
|
||||
base = "1" + _random_digits(11)
|
||||
return prefix + base + _checksum_ogrn(base)
|
||||
|
||||
|
||||
def generate_kpp(original: str) -> str:
|
||||
prefix = re.match(r'КПП\s*', original, re.IGNORECASE).group(0)
|
||||
return prefix + _random_digits(4) + random.choice(["01", "43", "77"]) + _random_digits(3)
|
||||
|
||||
|
||||
def generate_bik(original: str) -> str:
|
||||
prefix = re.match(r'БИК\s*', original, re.IGNORECASE).group(0)
|
||||
return prefix + "04" + _random_digits(7)
|
||||
|
||||
|
||||
def generate_rs(original: str) -> str:
|
||||
prefix = re.match(r'(?:р/с|расч[её]тный\s*сч[её]т)\s*', original, re.IGNORECASE).group(0)
|
||||
return prefix + "40702" + _random_digits(15)
|
||||
|
||||
|
||||
def generate_ks(original: str) -> str:
|
||||
prefix = re.match(r'(?:к/с|кор/сч|корр?[еи]?спондентский\s*сч[её]т)\s*', original, re.IGNORECASE).group(0)
|
||||
return prefix + "30101" + _random_digits(15)
|
||||
|
||||
|
||||
def generate_passport(original: str) -> str:
|
||||
m = re.match(r'(?:паспорт|серия\s+номер)[\s:№]*', original, re.IGNORECASE)
|
||||
prefix = m.group(0) if m else ""
|
||||
return prefix + f"{random.randint(10,99)} {random.randint(10,99)} {_random_digits(6)}"
|
||||
|
||||
|
||||
def generate_company(_: str) -> str:
|
||||
forms = ["ООО", "ЗАО", "АО"]
|
||||
nouns = ["Технология", "Прогресс", "Гарант", "Стандарт", "Импульс",
|
||||
"Вектор", "Сфера", "Альянс", "Синтез", "Меридиан", "Спектр", "Формат"]
|
||||
return f'{random.choice(forms)} «{random.choice(nouns)}»'
|
||||
|
||||
|
||||
def generate_person(_: str) -> str:
|
||||
s = random.choice(RU_SURNAMES)
|
||||
n = random.choice(RU_NAMES)
|
||||
p = random.choice(RU_PATRONYMICS)
|
||||
return f"{s} {n[0]}.{p[0]}."
|
||||
|
||||
|
||||
def generate_address(_: str) -> str:
|
||||
city = random.choice(RU_CITIES)
|
||||
street = random.choice(RU_STREETS)
|
||||
house = random.randint(1, 200)
|
||||
return f"{city}, ул. {street}, д. {house}"
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# Основной класс
|
||||
# ═══════════════════════════════════════════
|
||||
|
||||
class TwoPassObfuscator:
|
||||
"""Двухпроходный обфускатор: сбор сущностей → замена."""
|
||||
|
||||
def __init__(self, llm_client=None):
|
||||
self._mapping: Dict[str, str] = {} # оригинал → замена (только для точных текстовых совпадений)
|
||||
self._regex_replacements: List[Tuple[str, str, Callable]] = [] # (pattern, type, generator)
|
||||
self._sorted_keys: List[str] = [] # ключи mapping, отсортированные по длине (убывание)
|
||||
self._llm_client = llm_client
|
||||
|
||||
def obfuscate(self, files: List[Tuple[str, bytes, str]]) -> Tuple[bytes, str]:
|
||||
"""
|
||||
Главная точка входа.
|
||||
|
||||
Args:
|
||||
files: [(original_filename, content_bytes, content_type), ...]
|
||||
|
||||
Returns:
|
||||
(zip_bytes, mapping_csv_string)
|
||||
"""
|
||||
# --- Распаковать ZIP-файлы ---
|
||||
files = self._expand_zips(files)
|
||||
# --- Конвертировать PDF → DOCX ---
|
||||
files = self._convert_pdfs_to_docx(files)
|
||||
|
||||
try:
|
||||
# --- Проход 1: сбор сущностей ---
|
||||
all_texts: Dict[str, str] = {}
|
||||
all_docx: Dict[str, object] = {}
|
||||
|
||||
for fname, content, ctype in files:
|
||||
text, doc = self._extract_text(fname, content, ctype)
|
||||
all_texts[fname] = text
|
||||
if doc is not None:
|
||||
all_docx[fname] = doc
|
||||
if text and text != "[DOC binary — not parsed]":
|
||||
self._scan_regex(text)
|
||||
|
||||
if self._llm_client:
|
||||
self._scan_llm_ner(all_texts)
|
||||
|
||||
# Предсортировать ключи один раз для _apply_replacements
|
||||
self._sorted_keys = sorted(self._mapping.keys(), key=len, reverse=True)
|
||||
|
||||
# --- Проход 2: замена ---
|
||||
results = []
|
||||
for fname, content, ctype in files:
|
||||
obf_content = content
|
||||
if fname.endswith('.doc'):
|
||||
# .doc — бинарный, оставляем как есть
|
||||
pass
|
||||
elif fname in all_docx:
|
||||
obf_content = self._replace_in_docx(all_docx[fname])
|
||||
else:
|
||||
txt = all_texts.get(fname, '')
|
||||
obf_content = self._replace_in_text(txt, fname)
|
||||
results.append((fname, obf_content))
|
||||
|
||||
csv_str = self._build_mapping_csv()
|
||||
return self._build_zip(results, csv_str), csv_str
|
||||
|
||||
finally:
|
||||
self._mapping.clear()
|
||||
self._regex_replacements.clear()
|
||||
self._sorted_keys.clear()
|
||||
|
||||
def _convert_pdfs_to_docx(self, files: List[Tuple[str, bytes, str]]) -> List[Tuple[str, bytes, str]]:
|
||||
"""Конвертировать PDF в DOCX через pdfplumber. При совпадении имён — _из_pdf."""
|
||||
import pdfplumber
|
||||
from docx import Document as DocxDocument
|
||||
result = []
|
||||
existing_names = {f[0] for f in files}
|
||||
for fname, content, ctype in files:
|
||||
if not fname.lower().endswith('.pdf'):
|
||||
result.append((fname, content, ctype))
|
||||
continue
|
||||
try:
|
||||
doc = DocxDocument()
|
||||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||||
for page in pdf.pages:
|
||||
tables = page.extract_tables()
|
||||
for table in tables:
|
||||
if table:
|
||||
rows = [[str(c or "").strip() for c in (row or [])] for row in table]
|
||||
rows = [r for r in rows if any(r)]
|
||||
if rows:
|
||||
t = doc.add_table(rows=len(rows), cols=len(rows[0]))
|
||||
t.style = 'Table Grid'
|
||||
for ri, row in enumerate(rows):
|
||||
for ci, cell_text in enumerate(row):
|
||||
t.rows[ri].cells[ci].text = cell_text
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
for line in text.split('\n'):
|
||||
line = line.strip()
|
||||
if line:
|
||||
doc.add_paragraph(line)
|
||||
buf = io.BytesIO()
|
||||
doc.save(buf)
|
||||
new_name = fname[:-4] + '.docx'
|
||||
if new_name in existing_names:
|
||||
new_name = fname[:-4] + '_из_pdf.docx'
|
||||
existing_names.add(new_name)
|
||||
result.append((new_name, buf.getvalue(), ctype))
|
||||
except Exception as e:
|
||||
log.warning("PDF→DOCX error for %s: %s", fname, e)
|
||||
result.append((fname, content, ctype))
|
||||
return result
|
||||
|
||||
def _expand_zips(self, files: List[Tuple[str, bytes, str]]) -> List[Tuple[str, bytes, str]]:
|
||||
"""Распаковать ZIP-файлы, заменив их содержимым. Остальные файлы — как есть."""
|
||||
result = []
|
||||
for fname, content, ctype in files:
|
||||
if fname.lower().endswith('.zip'):
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(content)) as zf:
|
||||
total_size = sum(info.file_size for info in zf.infolist())
|
||||
if total_size > 500 * 1024 * 1024: # 500 MB
|
||||
log.warning("ZIP too large, skipping expansion: %s", fname)
|
||||
result.append((fname, content, ctype))
|
||||
continue
|
||||
if len(zf.infolist()) > 500:
|
||||
log.warning("ZIP too many files, skipping: %s", fname)
|
||||
result.append((fname, content, ctype))
|
||||
continue
|
||||
for info in zf.infolist():
|
||||
if info.is_dir():
|
||||
continue
|
||||
# cp437 → utf8 (как в services/unzip.py)
|
||||
name = info.filename
|
||||
try:
|
||||
name = name.encode("cp437").decode("utf-8", errors="replace")
|
||||
except (UnicodeDecodeError, UnicodeEncodeError):
|
||||
pass
|
||||
# Убрать path traversal
|
||||
name = os.path.basename(name)
|
||||
if not name or name.endswith("/") or ".." in name or "/" in name or "\\" in name:
|
||||
continue
|
||||
inner_data = zf.read(info)
|
||||
result.append((name, inner_data, ""))
|
||||
except Exception as e:
|
||||
log.warning("Failed to expand ZIP %s: %s", fname, e)
|
||||
result.append((fname, content, ctype))
|
||||
else:
|
||||
result.append((fname, content, ctype))
|
||||
return result
|
||||
|
||||
# --- Проход 1: обнаружение ---
|
||||
|
||||
def _extract_text(self, fname: str, content: bytes, ctype: str) -> Tuple[str, Optional[object]]:
|
||||
"""Извлечь текст из файла. Возвращает (text, docx_document_or_None)."""
|
||||
doc = None
|
||||
text = ""
|
||||
|
||||
ext = os.path.splitext(fname)[1].lower()
|
||||
|
||||
if ext == '.docx':
|
||||
try:
|
||||
from docx import Document
|
||||
except ImportError:
|
||||
text = content.decode('utf-8', errors='replace')
|
||||
return text, None
|
||||
doc = Document(io.BytesIO(content))
|
||||
text = "\n".join(p.text for p in doc.paragraphs)
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
text += "\n" + " | ".join(cell.text for cell in row.cells)
|
||||
|
||||
elif ext == '.pdf':
|
||||
try:
|
||||
import pdfplumber
|
||||
except ImportError:
|
||||
text = content.decode('utf-8', errors='replace')
|
||||
return text, None
|
||||
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
||||
for page in pdf.pages:
|
||||
t = page.extract_text()
|
||||
if t:
|
||||
text += t + "\n"
|
||||
for table in page.extract_tables():
|
||||
for row in table:
|
||||
text += "\n" + " | ".join(str(c) if c else "" for c in row)
|
||||
|
||||
elif ext == '.doc':
|
||||
# .doc — бинарный формат, без libreoffice не парсим
|
||||
# Пропускаем без изменений (не обфусцируем)
|
||||
text = "[DOC binary — not parsed]"
|
||||
return text, None
|
||||
|
||||
else:
|
||||
text = content.decode('utf-8', errors='replace')
|
||||
|
||||
return text, doc
|
||||
|
||||
def _scan_regex(self, text: str):
|
||||
"""Сканировать текст regex-паттернами, заполнить словарь замен."""
|
||||
for entity_type, pattern in ENTITY_PATTERNS.items():
|
||||
for match in re.finditer(pattern, text, re.IGNORECASE | re.MULTILINE):
|
||||
original = match.group(0).strip()
|
||||
if original and original not in self._mapping:
|
||||
generator = ENTITY_GENERATORS.get(entity_type, lambda x: "XXX")
|
||||
self._mapping[original] = generator(original)
|
||||
|
||||
# Компании (кроме НУБЕС — это Исполнитель)
|
||||
for match in COMPANY_PATTERN.finditer(text):
|
||||
original = match.group(0).strip()
|
||||
if original and original not in self._mapping:
|
||||
if re.search(r'НУБЕС|NUBES', original, re.IGNORECASE):
|
||||
continue # Исполнитель — не заменяем
|
||||
self._mapping[original] = generate_company(original)
|
||||
|
||||
# ФИО (Фамилия И.О.)
|
||||
for match in PERSON_PATTERN.finditer(text):
|
||||
original = match.group(0).strip()
|
||||
if original and original not in self._mapping:
|
||||
self._mapping[original] = generate_person(original)
|
||||
|
||||
def _scan_llm_ner(self, all_texts: Dict[str, str]):
|
||||
"""LLM NER для обнаружения имён и адресов во всех файлах."""
|
||||
combined = "\n\n---FILE---\n\n".join(
|
||||
f"FILE: {fname}\n{t[:3000]}" for fname, t in all_texts.items()
|
||||
)
|
||||
prompt = (
|
||||
"Ты — система обнаружения персональных данных в документах. "
|
||||
"Найди ВСЕ следующие сущности в тексте ниже:\n\n"
|
||||
"1. ФИО (полные и сокращённые — 'Иванов И.И.', 'Петров А.С.')\n"
|
||||
"2. Названия компаний-контрагентов (не 'НУБЕС')\n"
|
||||
"3. Почтовые адреса\n"
|
||||
"4. Паспортные данные\n\n"
|
||||
"Формат ответа — JSON-массив:\n"
|
||||
'[{"type": "person"|"company"|"address"|"passport", "value": "найденный текст"}]\n\n'
|
||||
f"Текст:\n{combined[:8000]}"
|
||||
)
|
||||
try:
|
||||
raw = self._llm_client.complete(prompt)
|
||||
import json
|
||||
# Игнорируем markdown-обёртку
|
||||
raw = raw.strip()
|
||||
if raw.startswith("```"):
|
||||
raw = raw.split("\n", 1)[1]
|
||||
if raw.endswith("```"):
|
||||
raw = raw[:-3]
|
||||
entities = json.loads(raw)
|
||||
for ent in entities:
|
||||
val = ent.get("value", "").strip()
|
||||
if val and val not in self._mapping:
|
||||
if ent.get("type") == "person":
|
||||
self._mapping[val] = generate_person(val)
|
||||
elif ent.get("type") == "company":
|
||||
self._mapping[val] = generate_company(val)
|
||||
elif ent.get("type") == "address":
|
||||
self._mapping[val] = generate_address(val)
|
||||
elif ent.get("type") == "passport":
|
||||
self._mapping[val] = generate_passport(val)
|
||||
except Exception as e:
|
||||
log.warning("LLM NER failed: %s", e)
|
||||
|
||||
# --- Проход 2: замена ---
|
||||
|
||||
def _replace_in_docx(self, doc) -> bytes:
|
||||
"""Заменить сущности в docx. Склеиваем runs → заменяем → пишем в первый run, очищаем остальные."""
|
||||
for para in doc.paragraphs:
|
||||
if not para.runs:
|
||||
continue
|
||||
full_text = "".join(run.text for run in para.runs)
|
||||
replaced = self._apply_replacements(full_text)
|
||||
if replaced != full_text:
|
||||
para.runs[0].text = replaced
|
||||
for run in para.runs[1:]:
|
||||
run.text = ""
|
||||
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
for para in cell.paragraphs:
|
||||
if not para.runs:
|
||||
continue
|
||||
full_text = "".join(run.text for run in para.runs)
|
||||
replaced = self._apply_replacements(full_text)
|
||||
if replaced != full_text:
|
||||
para.runs[0].text = replaced
|
||||
for run in para.runs[1:]:
|
||||
run.text = ""
|
||||
|
||||
buf = io.BytesIO()
|
||||
doc.save(buf)
|
||||
return buf.getvalue()
|
||||
|
||||
def _replace_in_text(self, text: str, fname: str) -> bytes:
|
||||
"""Заменить сущности в plain text (для PDF и прочих)."""
|
||||
replaced = self._apply_replacements(text)
|
||||
# Для PDF пока отдаём текст (MVP — без сохранения форматирования PDF)
|
||||
return replaced.encode('utf-8')
|
||||
|
||||
def _apply_replacements(self, text: str) -> str:
|
||||
"""Применить все замены из словаря mapping к строке. Сначала длинные, потом короткие."""
|
||||
result = text
|
||||
for original in self._sorted_keys:
|
||||
replacement = self._mapping[original]
|
||||
# Границы слова — если сущность начинается и заканчивается на \w
|
||||
if original and original[0].isalnum() and original[-1].isalnum():
|
||||
pattern = r'(?<!\w)' + re.escape(original) + r'(?!\w)'
|
||||
else:
|
||||
pattern = re.escape(original)
|
||||
result = re.sub(pattern, replacement, result)
|
||||
return result
|
||||
|
||||
# --- Сборка выдачи ---
|
||||
|
||||
def _build_zip(self, files: List[Tuple[str, bytes]], mapping_csv: str = "") -> bytes:
|
||||
"""Собрать ZIP с обфусцированными файлами + mapping.csv."""
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
for fname, content in files:
|
||||
info = zipfile.ZipInfo(fname)
|
||||
info.flag_bits |= 0x800 # UTF-8 filename
|
||||
zf.writestr(info, content)
|
||||
if mapping_csv:
|
||||
zf.writestr("mapping.csv", '\ufeff'.encode('utf-8') + mapping_csv.encode('utf-8'))
|
||||
return buf.getvalue()
|
||||
|
||||
def _build_mapping_csv(self) -> str:
|
||||
"""Собрать mapping.csv."""
|
||||
buf = io.StringIO()
|
||||
writer = csv.writer(buf)
|
||||
writer.writerow(["тип_данных", "оригинал", "замена"])
|
||||
for original, replacement in sorted(self._mapping.items()):
|
||||
etype = "text"
|
||||
# inn_fl ДО inn_ul (12 цифр vs 10)
|
||||
for t in ["phone", "email", "inn_fl", "inn_ul", "ogrn", "kpp", "bik", "rs", "ks", "passport"]:
|
||||
pat = ENTITY_PATTERNS.get(t, "")
|
||||
if pat and re.match(pat, original, re.IGNORECASE):
|
||||
etype = t
|
||||
break
|
||||
if COMPANY_PATTERN.match(original):
|
||||
etype = "company"
|
||||
writer.writerow([etype, original, replacement])
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════
|
||||
# Маппинг entity_type → генератор
|
||||
# ═══════════════════════════════════════════
|
||||
|
||||
ENTITY_GENERATORS: Dict[str, Callable] = {
|
||||
"phone": generate_phone,
|
||||
"email": generate_email,
|
||||
"inn_ul": generate_inn10,
|
||||
"inn_fl": generate_inn12,
|
||||
"ogrn": generate_ogrn,
|
||||
"kpp": generate_kpp,
|
||||
"bik": generate_bik,
|
||||
"rs": generate_rs,
|
||||
"ks": generate_ks,
|
||||
"passport": generate_passport,
|
||||
}
|
||||
|
||||
|
||||
def obfuscate_files(files: List[Tuple[str, bytes, str]], llm_client=None) -> Tuple[bytes, str]:
|
||||
"""Удобная функция: обфусцировать список файлов → (zip_bytes, csv_string)."""
|
||||
obf = TwoPassObfuscator(llm_client=llm_client)
|
||||
return obf.obfuscate(files)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""
|
||||
Grouping service — match classified documents into contract groups.
|
||||
|
||||
Архитектурное решение (Opus, Q4 + Q6):
|
||||
- Двухпроходный гибрид: LLM извлекает строки (classify.py), Python нормализует и группирует.
|
||||
- Нормализация номеров: uppercase + только буквы/цифры.
|
||||
"МЭС-123-2024" == "МЭС 123/2024" после нормализации.
|
||||
- Группировка на бэкенде (не на фронте): Python regex/unicode надёжнее JS.
|
||||
- apply_groups(): создаёт contracts + supplements с авто-порядком по дате.
|
||||
"""
|
||||
import re
|
||||
from site.db import documents as db_docs
|
||||
from site.db import contracts as db_contracts
|
||||
from site.db import supplements as db_supplements
|
||||
|
||||
|
||||
def normalize_number(num):
|
||||
"""
|
||||
Нормализация номера договора для сравнения.
|
||||
Убирает всё кроме букв и цифр, приводит к uppercase.
|
||||
Пример: "МЭС-123-2024" → "МЭС1232024", "МЭС 123/2024" → "МЭС1232024".
|
||||
"""
|
||||
if not num:
|
||||
return ""
|
||||
return re.sub(r"[^A-Z0-9А-Я]", "", num.upper())
|
||||
|
||||
|
||||
def group_documents(batch_id):
|
||||
"""
|
||||
Сгруппировать классифицированные документы по контрактам.
|
||||
|
||||
Алгоритм:
|
||||
1. Отделить contract от supplement/specification
|
||||
2. Каждый contract → якорь группы
|
||||
3. Для каждого supplement: найти contract по parent_number (нормализованный)
|
||||
4. Оставшиеся supplement/spec → виртуальные группы по parent_number/own_number
|
||||
5. Совсем без номеров → группа "__unresolved__"
|
||||
6. Внутри группы сортировка по doc_date
|
||||
|
||||
Возвращает {ok, groups: [{contract_number, counterparty, documents: [...]}], total_docs}.
|
||||
"""
|
||||
docs = db_docs.list_by_batch(batch_id)
|
||||
classified = [d for d in docs if d.get("classify_status") == "classified"]
|
||||
|
||||
# Separate contracts and supplements
|
||||
contracts_list = []
|
||||
supplements_list = []
|
||||
|
||||
for d in classified:
|
||||
if d.get("doc_type") == "contract":
|
||||
contracts_list.append(d)
|
||||
else:
|
||||
supplements_list.append(d)
|
||||
|
||||
groups = []
|
||||
|
||||
# Each contract becomes a group
|
||||
for c in contracts_list:
|
||||
group = {
|
||||
"contract_number": c.get("own_number") or c.get("filename", ""),
|
||||
"counterparty": c.get("counterparty") or "",
|
||||
"documents": [c],
|
||||
}
|
||||
# Find supplements matching this contract
|
||||
c_norm = normalize_number(c.get("own_number"))
|
||||
for s in supplements_list:
|
||||
parent = normalize_number(s.get("parent_number") or "")
|
||||
own = normalize_number(s.get("own_number") or "")
|
||||
if parent == c_norm or own == c_norm:
|
||||
if s not in group["documents"]:
|
||||
group["documents"].append(s)
|
||||
|
||||
# Sort by date
|
||||
group["documents"].sort(key=lambda x: x.get("doc_date") or "")
|
||||
|
||||
# Remove matched supplements from the pool
|
||||
for s in group["documents"]:
|
||||
if s in supplements_list:
|
||||
supplements_list.remove(s)
|
||||
|
||||
groups.append(group)
|
||||
|
||||
# ── Virtual groups: unmatched supplements grouped by number ──────────
|
||||
# Group remaining supplements/specs by normalized parent_number (priority) or own_number
|
||||
virtual = {}
|
||||
for s in supplements_list:
|
||||
num = normalize_number(s.get("parent_number") or s.get("own_number") or "")
|
||||
if not num:
|
||||
continue # no number → stays in supplements_list for unresolved
|
||||
if num not in virtual:
|
||||
# Use counterparty from first doc in group
|
||||
cp = s.get("counterparty") or ""
|
||||
virtual[num] = {
|
||||
"contract_number": s.get("parent_number") or s.get("own_number") or "?",
|
||||
"counterparty": cp,
|
||||
"documents": [],
|
||||
}
|
||||
virtual[num]["documents"].append(s)
|
||||
# Update counterparty if current doc has a better one
|
||||
if not virtual[num]["counterparty"] and s.get("counterparty"):
|
||||
virtual[num]["counterparty"] = s.get("counterparty")
|
||||
|
||||
for vnum, vgroup in virtual.items():
|
||||
vgroup["documents"].sort(key=lambda x: x.get("doc_date") or "")
|
||||
groups.append(vgroup)
|
||||
# Remove grouped docs from supplements_list
|
||||
for s in vgroup["documents"]:
|
||||
supplements_list.remove(s)
|
||||
|
||||
# ── Unresolved: everything left (no number, failed, pending, other) ──
|
||||
unmatched = [s for s in supplements_list] # remaining after virtual grouping
|
||||
unmatched += [d for d in docs if d.get("classify_status") != "classified"]
|
||||
|
||||
if unmatched:
|
||||
groups.append({
|
||||
"contract_number": "__unresolved__",
|
||||
"counterparty": "",
|
||||
"documents": unmatched,
|
||||
})
|
||||
|
||||
return {"ok": True, "groups": groups, "total_docs": len(docs)}
|
||||
|
||||
|
||||
def apply_groups(batch_id, groups_data):
|
||||
"""
|
||||
Применить подтверждённые группы: создать contracts + supplements.
|
||||
|
||||
Вызывается из POST /api/apply-groups.
|
||||
Для каждой группы (кроме __unresolved__):
|
||||
1. Создать запись в contracts (number, client)
|
||||
2. Для каждого документа создать supplement (type='initial' для первого, 'additional' для остальных)
|
||||
3. Порядок supplements соответствует порядку документов в группе (сортировка по дате уже сделана)
|
||||
|
||||
Возвращает {ok, created: количество созданных supplements}.
|
||||
"""
|
||||
created = 0
|
||||
contract_ids = []
|
||||
for g in groups_data:
|
||||
contract_number = g.get("contract_number", "")
|
||||
if contract_number == "__unresolved__":
|
||||
continue
|
||||
counterparty = g.get("counterparty", "")
|
||||
docs = g.get("documents", [])
|
||||
|
||||
try:
|
||||
c = db_contracts.insert(contract_number, counterparty)
|
||||
contract_id = c["id"]
|
||||
contract_ids.append(contract_id)
|
||||
|
||||
for i, d in enumerate(docs):
|
||||
doc_id = d.get("id")
|
||||
if not doc_id:
|
||||
continue
|
||||
supp_type = "initial" if i == 0 else "additional"
|
||||
db_supplements.insert(contract_id, doc_id, supp_type)
|
||||
created += 1
|
||||
except Exception as e:
|
||||
return {"ok": False, "error": f"apply_groups failed at {contract_number}: {e}"}
|
||||
|
||||
return {"ok": True, "created": created, "contract_ids": contract_ids}
|
||||
@@ -0,0 +1,35 @@
|
||||
"""LLM service — call LLM API with optional DI."""
|
||||
import os, json
|
||||
|
||||
LLM_URL = "https://api.aillm.ru/v1/chat/completions"
|
||||
LLM_KEY = os.environ.get("LLM_KEY") or os.environ.get("LLM_API_KEY", "")
|
||||
LLM_MODEL = "gpt-oss-120b"
|
||||
|
||||
# Ленивый singleton — обратная совместимость
|
||||
_llm_client = None
|
||||
|
||||
|
||||
def _get_default_client():
|
||||
global _llm_client
|
||||
if _llm_client is None:
|
||||
from site.services.llm_client import HttpxLLMClient
|
||||
_llm_client = HttpxLLMClient(url=LLM_URL, key=LLM_KEY, model=LLM_MODEL)
|
||||
return _llm_client
|
||||
|
||||
|
||||
def call_llm(current_spec, doc_text, build_prompt_fn, llm_client=None):
|
||||
"""Call LLM. Returns (parsed_result, prompt_id).
|
||||
llm_client: LLMClient (optional). Default — HttpxLLMClient (prod).
|
||||
"""
|
||||
if llm_client is None:
|
||||
llm_client = _get_default_client()
|
||||
|
||||
prompt, prompt_id = build_prompt_fn(current_spec, doc_text)
|
||||
raw_text = llm_client.complete(prompt)
|
||||
|
||||
json_text = raw_text
|
||||
if "```json" in json_text:
|
||||
json_text = json_text.split("```json")[1].split("```")[0]
|
||||
elif "```" in json_text:
|
||||
json_text = json_text.split("```")[1].split("```")[0]
|
||||
return json.loads(json_text.strip()), prompt_id
|
||||
@@ -0,0 +1,74 @@
|
||||
"""LLM Client — протокол + httpx-реализация + fake для тестов.
|
||||
|
||||
План decoupling Ф2:
|
||||
- LLMClient.complete(prompt) -> str — единственный метод
|
||||
- Парсинг JSON остаётся у вызывающего (call_llm / _call_llm_classify)
|
||||
- HttpxLLMClient — продакшен
|
||||
- FakeLLMClient — тесты (возвращает сохранённые ответы)
|
||||
"""
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class LLMClient(Protocol):
|
||||
"""Протокол LLM-клиента. Один метод — сырой вызов."""
|
||||
def complete(self, prompt: str) -> str:
|
||||
"""Отправить prompt в LLM, вернуть raw_text."""
|
||||
...
|
||||
|
||||
|
||||
class HttpxLLMClient:
|
||||
"""Продакшен-клиент через httpx → api.aillm.ru."""
|
||||
|
||||
def __init__(self, url: str, key: str, model: str = "gpt-oss-120b",
|
||||
max_tokens: int = 8000, temperature: float = 0.1, timeout: int = 120):
|
||||
self.url = url
|
||||
self.key = key
|
||||
self.model = model
|
||||
self.max_tokens = max_tokens
|
||||
self.temperature = temperature
|
||||
self.timeout = timeout
|
||||
|
||||
def complete(self, prompt: str) -> str:
|
||||
import httpx
|
||||
payload = {
|
||||
"model": self.model,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": self.max_tokens,
|
||||
"temperature": self.temperature,
|
||||
}
|
||||
with httpx.Client(http2=True, timeout=self.timeout, verify=True) as client:
|
||||
resp = client.post(
|
||||
self.url,
|
||||
json=payload,
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return data["choices"][0]["message"]["content"]
|
||||
|
||||
|
||||
class FakeLLMClient:
|
||||
"""Тестовый клиент — возвращает сохранённые ответы из словаря."""
|
||||
|
||||
def __init__(self, responses: dict = None):
|
||||
self.responses = responses or {}
|
||||
self.calls: list[str] = [] # история вызовов для проверок
|
||||
|
||||
def add(self, prompt_key: str, response: str):
|
||||
"""Зарегистрировать ответ для конкретного prompt_key."""
|
||||
self.responses[prompt_key] = response
|
||||
|
||||
def complete(self, prompt: str) -> str:
|
||||
self.calls.append(prompt)
|
||||
# Ищем точное совпадение
|
||||
if prompt in self.responses:
|
||||
return self.responses[prompt]
|
||||
# Ищем по частичному ключу
|
||||
for key, resp in self.responses.items():
|
||||
if key in prompt:
|
||||
return resp
|
||||
# Fallback
|
||||
return self.responses.get("default", '{"error": "no response registered"}')
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Metrics — quality signals without golden dataset.
|
||||
|
||||
Checks that work immediately:
|
||||
- Arithmetic: sum == price * qty (free signal of LLM/data errors)
|
||||
- JSON fix rate: how often _safe_json_parse has to repair LLM output
|
||||
"""
|
||||
from decimal import Decimal, InvalidOperation
|
||||
|
||||
|
||||
def check_arithmetic(ops: list) -> list[dict]:
|
||||
"""Check sum == price * qty for ADD/UPDATE operations.
|
||||
Returns list of mismatches: [{action, name, price, qty, expected_sum, actual_sum, diff}]
|
||||
"""
|
||||
mismatches = []
|
||||
for op in ops:
|
||||
action = op.get("action", "")
|
||||
if action not in ("ADD", "UPDATE"):
|
||||
continue
|
||||
|
||||
row = op.get("new_row") or op.get("new_values") or {}
|
||||
price = _to_decimal(row.get("price"))
|
||||
qty = _to_decimal(row.get("qty"))
|
||||
actual_sum = _to_decimal(row.get("sum"))
|
||||
|
||||
if price is None or qty is None or actual_sum is None:
|
||||
continue # can't check without all three
|
||||
|
||||
expected = price * qty
|
||||
if expected != actual_sum:
|
||||
mismatches.append({
|
||||
"action": action,
|
||||
"name": row.get("name", "")[:100],
|
||||
"price": float(price),
|
||||
"qty": float(qty),
|
||||
"expected_sum": float(expected),
|
||||
"actual_sum": float(actual_sum),
|
||||
"diff": float(actual_sum - expected),
|
||||
})
|
||||
return mismatches
|
||||
|
||||
|
||||
class ClassifyMetrics:
|
||||
"""Track _safe_json_parse fix rate per batch."""
|
||||
def __init__(self):
|
||||
self.total = 0
|
||||
self.fixes = 0 # how many times JSON needed repair
|
||||
|
||||
def record(self, needed_fix: bool):
|
||||
self.total += 1
|
||||
if needed_fix:
|
||||
self.fixes += 1
|
||||
|
||||
@property
|
||||
def fix_rate(self) -> float:
|
||||
return self.fixes / self.total if self.total else 0.0
|
||||
|
||||
def summary(self) -> dict:
|
||||
return {
|
||||
"total_classifications": self.total,
|
||||
"json_fixes": self.fixes,
|
||||
"json_fix_rate": round(self.fix_rate, 3),
|
||||
}
|
||||
|
||||
|
||||
def _to_decimal(val) -> Decimal | None:
|
||||
"""Safe conversion to Decimal with number normalization."""
|
||||
if val is None or val == "":
|
||||
return None
|
||||
try:
|
||||
s = str(val).replace("\xa0", "").replace(" ", "").replace(",", ".")
|
||||
return Decimal(s)
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def normalize_date(ds: str) -> str | None:
|
||||
"""Normalize date to YYYY-MM-DD. Handles DD.MM.YYYY, YYYY-MM-DD, etc."""
|
||||
if not ds:
|
||||
return None
|
||||
import re
|
||||
# DD.MM.YYYY → YYYY-MM-DD
|
||||
m = re.match(r"(\d{2})\.(\d{2})\.(\d{4})", ds)
|
||||
if m:
|
||||
return f"{m.group(3)}-{m.group(2)}-{m.group(1)}"
|
||||
# YYYY-MM-DD — already canonical
|
||||
if re.match(r"\d{4}-\d{2}-\d{2}", ds):
|
||||
return ds
|
||||
return ds # return as-is if unrecognized format
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Parse service — PDF (pdfplumber), DOCX (python-docx), plain text fallback."""
|
||||
import io
|
||||
|
||||
|
||||
def parse_file(filename, data):
|
||||
"""Parse file bytes → {status, elements, element_count}."""
|
||||
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||
try:
|
||||
if ext == "pdf":
|
||||
return _parse_pdf(data)
|
||||
elif ext == "docx":
|
||||
return _parse_docx(data)
|
||||
elif ext == "doc":
|
||||
return _parse_docx(data) # python-docx handles most .doc
|
||||
else:
|
||||
return _parse_text(data)
|
||||
except Exception as e:
|
||||
return {"status": "error", "error": str(e), "element_count": 0}
|
||||
|
||||
|
||||
def _parse_pdf(data):
|
||||
"""Parse PDF with pdfplumber — preserves table structure (unlike PyPDF2)."""
|
||||
import pdfplumber
|
||||
elements = []
|
||||
with pdfplumber.open(io.BytesIO(data)) as pdf:
|
||||
for page in pdf.pages:
|
||||
# Tables first — preserves column structure critical for specs
|
||||
tables = page.extract_tables()
|
||||
for table in tables:
|
||||
if table:
|
||||
rows = []
|
||||
for row in table:
|
||||
if row:
|
||||
cells = [str(cell or "").strip() for cell in row]
|
||||
if any(cells):
|
||||
rows.append(cells)
|
||||
if rows:
|
||||
elements.append({"type": "table", "rows": rows})
|
||||
# Remaining text as paragraphs
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
for line in text.split("\n"):
|
||||
line = line.strip()
|
||||
if line:
|
||||
elements.append({"type": "paragraph", "text": line, "style": ""})
|
||||
return {"status": "parsed", "element_count": len(elements), "elements": elements}
|
||||
|
||||
|
||||
def _parse_docx(data):
|
||||
import docx
|
||||
doc = docx.Document(io.BytesIO(data))
|
||||
elements = []
|
||||
|
||||
for block in doc.element.body:
|
||||
tag = block.tag.split("}")[-1] if "}" in block.tag else block.tag
|
||||
if tag == "p":
|
||||
text = _extract_paragraph_text(block)
|
||||
if text:
|
||||
elements.append({"type": "paragraph", "text": text, "style": ""})
|
||||
elif tag == "tbl":
|
||||
rows = []
|
||||
for tr in block.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tr"):
|
||||
cells = []
|
||||
for tc in tr.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tc"):
|
||||
cell_text = "".join(t.text or "" for t in tc.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t"))
|
||||
cells.append(cell_text.strip())
|
||||
if cells:
|
||||
rows.append(cells)
|
||||
if rows:
|
||||
elements.append({"type": "table", "rows": rows})
|
||||
|
||||
return {"status": "parsed", "element_count": len(elements), "elements": elements}
|
||||
|
||||
|
||||
def _extract_paragraph_text(p_elem):
|
||||
texts = []
|
||||
for t in p_elem.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t"):
|
||||
if t.text:
|
||||
texts.append(t.text)
|
||||
return "".join(texts).strip()
|
||||
|
||||
|
||||
def _parse_text(data):
|
||||
text = data.decode("utf-8", errors="ignore")[:5000]
|
||||
lines = [l.strip() for l in text.split("\n") if l.strip()]
|
||||
return {"status": "parsed", "element_count": len(lines),
|
||||
"elements": [{"type": "paragraph", "text": l, "style": ""} for l in lines]}
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Process service — SSE pipeline: reset → supplements → LLM → apply.
|
||||
Адаптирован из deploy/compare/process.py: callback → generator.
|
||||
"""
|
||||
import json, time
|
||||
from site.db import supplements, spec_current, spec_events
|
||||
from site.services.metrics import check_arithmetic
|
||||
|
||||
|
||||
def run_pipeline(contract_id, order_ids, build_prompt_fn):
|
||||
"""Generator: yield SSE events вместо sse_send callback.
|
||||
|
||||
Использование:
|
||||
for event in run_pipeline(cid, order_ids, build_prompt):
|
||||
yield f"data: {json.dumps(event)}\\n\\n"
|
||||
"""
|
||||
t0 = time.time()
|
||||
|
||||
# 0. Reset
|
||||
spec_events.reset(contract_id)
|
||||
|
||||
# 1. Get supplements with parsed documents
|
||||
supps = supplements.list_by_contract(contract_id)
|
||||
if order_ids:
|
||||
order_list = [x.strip() for x in order_ids.split(",") if x.strip()]
|
||||
order_map = {oid: i for i, oid in enumerate(order_list)}
|
||||
supps.sort(key=lambda s: order_map.get(s["id"], 999999))
|
||||
else:
|
||||
supps.sort(key=lambda s: (s.get("doc_date") or "9999-99-99", s.get("created_at", "")))
|
||||
|
||||
if not supps:
|
||||
yield {"type": "error", "message": "Нет распарсенных файлов"}
|
||||
return
|
||||
|
||||
from site.services.llm import call_llm
|
||||
|
||||
for s in supps:
|
||||
sid = s["id"]
|
||||
filename = s.get("filename", "?")
|
||||
|
||||
# Current spec
|
||||
cur = spec_current.list_by_contract(contract_id)
|
||||
current_spec = []
|
||||
for r in cur:
|
||||
current_spec.append({
|
||||
"hash": r["name_hash"],
|
||||
"name": r["name"],
|
||||
"price": float(r["price"]) if r.get("price") is not None else None,
|
||||
"qty": float(r["qty"]) if r.get("qty") is not None else None,
|
||||
"sum": float(r["sum"]) if r.get("sum") is not None else None,
|
||||
"date_start": r["date_start"],
|
||||
})
|
||||
|
||||
# Get elements_json
|
||||
ej = spec_current.get_elements_json(s["document_id"])
|
||||
if not ej:
|
||||
yield {
|
||||
"type": "extract_error",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
"error": "no elements_json",
|
||||
}
|
||||
continue
|
||||
|
||||
# Build doc text from elements
|
||||
doc_text = _elements_to_text(ej)
|
||||
|
||||
yield {
|
||||
"type": "extract_start",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
}
|
||||
|
||||
# LLM call
|
||||
try:
|
||||
t1 = time.time()
|
||||
result, prompt_id = call_llm(current_spec, doc_text, build_prompt_fn)
|
||||
ops = result.get("ops", [])
|
||||
mode = result.get("mode", "llm")
|
||||
|
||||
yield {
|
||||
"type": "llm_done",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
"ops_count": len(ops),
|
||||
"mode": mode,
|
||||
"time_s": round(time.time() - t1, 1),
|
||||
}
|
||||
|
||||
# Apply ops to DB
|
||||
applied_ops = []
|
||||
summary = {"added": 0, "updated": 0, "deleted": 0, "unresolved": 0}
|
||||
for op in ops:
|
||||
action = op.get("action", "UNRESOLVED")
|
||||
summary[action.lower()] = summary.get(action.lower(), 0) + 1
|
||||
try:
|
||||
if action == "ADD":
|
||||
nr = op.get("new_row", {})
|
||||
spec_events.add_row(contract_id, sid, nr)
|
||||
elif action == "UPDATE":
|
||||
nr = op.get("new_row", {})
|
||||
nv = op.get("new_values", {})
|
||||
target = op.get("target_hash", "")
|
||||
spec_events.update_row(contract_id, sid, target, nr, nv)
|
||||
elif action == "DELETE":
|
||||
target = op.get("target_hash", "")
|
||||
spec_events.delete_row(contract_id, sid, target)
|
||||
applied_ops.append(op)
|
||||
except Exception:
|
||||
summary["unresolved"] = summary.get("unresolved", 0) + 1
|
||||
|
||||
# Arithmetic check
|
||||
check_arithmetic(contract_id)
|
||||
|
||||
yield {
|
||||
"type": "applied",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
"summary": summary,
|
||||
"ops": applied_ops,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
yield {
|
||||
"type": "extract_error",
|
||||
"supplement_id": sid,
|
||||
"filename": filename,
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
total_time = round(time.time() - t0, 1)
|
||||
yield {"type": "complete", "total_time_s": total_time}
|
||||
|
||||
|
||||
def _elements_to_text(ej):
|
||||
"""Extract flat text from elements_json for LLM prompt."""
|
||||
if isinstance(ej, dict) and "Value" in ej:
|
||||
ej = ej["Value"]
|
||||
if isinstance(ej, str):
|
||||
try:
|
||||
ej = json.loads(ej)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
return ej
|
||||
|
||||
lines = []
|
||||
if isinstance(ej, list):
|
||||
for el in ej:
|
||||
if isinstance(el, dict):
|
||||
if el.get("type") == "paragraph":
|
||||
lines.append(el.get("text", ""))
|
||||
elif el.get("type") == "table":
|
||||
for row in el.get("rows", []):
|
||||
lines.append(" | ".join(str(c) for c in row))
|
||||
elif isinstance(el, str):
|
||||
lines.append(el)
|
||||
return "\n".join(lines)
|
||||
Reference in New Issue
Block a user