535 lines
24 KiB
Python
535 lines
24 KiB
Python
"""
|
||
Grouping service — match classified documents into contract groups.
|
||
|
||
Архитектурное решение (Opus, Q4 + Q6):
|
||
- Двухпроходный гибрид: LLM извлекает строки (classify.py), Python нормализует и группирует.
|
||
- Нормализация номеров: uppercase + только буквы/цифры.
|
||
"МЭС-123-2024" == "МЭС 123/2024" после нормализации.
|
||
- Группировка на бэкенде (не на фронте): Python regex/unicode надёжнее JS.
|
||
- apply_groups(): создаёт contracts + supplements с авто-порядком по дате.
|
||
"""
|
||
import re
|
||
from db import documents as db_docs
|
||
from db import contracts as db_contracts
|
||
from db import supplements as db_supplements
|
||
from db import dossiers as db_dossiers
|
||
from db import exceptions as db_exceptions
|
||
from db import extractions as db_extractions
|
||
from db.connection import transaction
|
||
|
||
|
||
_HOMOGLYPHS = str.maketrans({
|
||
"A": "А", "B": "В", "C": "С", "E": "Е", "H": "Н", "K": "К",
|
||
"M": "М", "O": "О", "P": "Р", "T": "Т", "X": "Х", "Y": "У",
|
||
})
|
||
|
||
|
||
def normalize_number(num):
|
||
"""
|
||
Нормализация номера договора для сравнения.
|
||
Убирает всё кроме букв и цифр, приводит к uppercase.
|
||
Пример: "МЭС-123-2024" → "МЭС1232024", "МЭС 123/2024" → "МЭС1232024".
|
||
"""
|
||
if not num:
|
||
return ""
|
||
return re.sub(r"[^A-Z0-9А-Я]", "", num.upper())
|
||
|
||
|
||
def canonicalize_contract_number(number):
|
||
"""Нормализовать полный номер, включая визуальные латинско-кириллические пары."""
|
||
if not number:
|
||
return ""
|
||
normalized = str(number).upper().translate(_HOMOGLYPHS)
|
||
return re.sub(r"[^0-9А-ЯЁ]", "", normalized)
|
||
|
||
|
||
def _is_contract_number(number):
|
||
"""Отличить номер договора от короткого номера приложения или допсоглашения."""
|
||
key = canonicalize_contract_number(number)
|
||
return len(key) >= 4 and not (key.isdigit() and int(key) < 1000)
|
||
|
||
|
||
def _contract_number_identity(number):
|
||
"""Вернуть полный ключ и ограниченный alias-ключ для формата XXXnnn-number.
|
||
|
||
Короткое ядро допускается только при наличии явного клиентского префикса и
|
||
минимум четырёх цифр, чтобы случайные хвосты вроде «1» не объединяли договоры.
|
||
Пары двух разных prefixed-номеров по одному ядру не считаются совпадением.
|
||
"""
|
||
full_key = canonicalize_contract_number(number)
|
||
normalized = str(number or "").upper().translate(_HOMOGLYPHS)
|
||
prefix_match = re.match(r"^(?:XXX|ХХХ)\d+[-_/](.+)$", normalized)
|
||
if not prefix_match:
|
||
suffix_aliases = [full_key]
|
||
numbered_suffix = re.match(r"^(.+)_\d+$", normalized)
|
||
if numbered_suffix:
|
||
core_key = canonicalize_contract_number(numbered_suffix.group(1))
|
||
if sum(character.isdigit() for character in core_key) >= 4:
|
||
suffix_aliases.append(core_key)
|
||
return {
|
||
"full": full_key, "suffix": full_key,
|
||
"suffix_aliases": suffix_aliases, "prefixed": False,
|
||
}
|
||
suffix_key = canonicalize_contract_number(prefix_match.group(1))
|
||
digit_count = sum(character.isdigit() for character in suffix_key)
|
||
if digit_count < 4:
|
||
return {
|
||
"full": full_key, "suffix": full_key,
|
||
"suffix_aliases": [full_key], "prefixed": True,
|
||
}
|
||
return {
|
||
"full": full_key, "suffix": suffix_key,
|
||
"suffix_aliases": [suffix_key], "prefixed": True,
|
||
}
|
||
|
||
|
||
def canonicalize_counterparty(name, inn):
|
||
"""Вернуть ключи контрагента для сравнения с жёстким ограничением по ИНН."""
|
||
inn_key = re.sub(r"\D", "", str(inn or ""))
|
||
if inn_key and len(inn_key) != 10 and len(inn_key) != 12:
|
||
inn_key = ""
|
||
normalized_name = str(name or "").upper().translate(_HOMOGLYPHS)
|
||
normalized_name = re.sub(r"[«»\"'`.,()\s]", "", normalized_name)
|
||
normalized_name = re.sub(r"^(ООО|ЗАО|АО|ПАО|ИП)", "", normalized_name)
|
||
return {"name": normalized_name, "inn": inn_key}
|
||
|
||
|
||
def _counterparties_compatible(left, right):
|
||
"""Запретить связь только при конфликте двух явно указанных ИНН."""
|
||
left_id = canonicalize_counterparty(left.get("name"), left.get("inn"))
|
||
right_id = canonicalize_counterparty(right.get("name"), right.get("inn"))
|
||
return not (left_id["inn"] and right_id["inn"] and left_id["inn"] != right_id["inn"])
|
||
|
||
|
||
def _passport_keys(passport):
|
||
"""Собрать ключи родительских договоров, отфильтровав номера приложений."""
|
||
doc_class = passport.get("doc_class")
|
||
values = []
|
||
own_number = passport.get("own_number")
|
||
if doc_class == "contract_frame" and _is_contract_number(own_number):
|
||
values.append((own_number, "own_number"))
|
||
elif doc_class == "specification" and _is_contract_number(own_number):
|
||
values.append((passport["own_number"], "own_number"))
|
||
for reference in passport.get("referenced_numbers") or []:
|
||
if isinstance(reference, dict) and reference.get("number"):
|
||
values.append((reference["number"], reference.get("source") or "referenced_contract"))
|
||
unique = {}
|
||
for raw, source in values:
|
||
identity = _contract_number_identity(raw)
|
||
key = identity["full"]
|
||
if key and _is_contract_number(key):
|
||
unique.setdefault(key, {"raw": str(raw), "source": str(source), **identity})
|
||
return unique
|
||
|
||
|
||
def _shared_contract_match(left_keys, right_keys):
|
||
"""Найти безопасное совпадение полного ключа или его допустимого ядра."""
|
||
for left_key, left in left_keys.items():
|
||
for right_key, right in right_keys.items():
|
||
if left_key == right_key:
|
||
return left_key, left, right
|
||
if left.get("prefixed") and not right.get("prefixed"):
|
||
same_suffix = left.get("suffix") in right.get("suffix_aliases", [])
|
||
elif right.get("prefixed") and not left.get("prefixed"):
|
||
same_suffix = right.get("suffix") in left.get("suffix_aliases", [])
|
||
else:
|
||
same_suffix = left.get("suffix") == right.get("suffix")
|
||
if same_suffix and (left.get("prefixed") or right.get("prefixed")):
|
||
if left.get("prefixed") and right.get("prefixed"):
|
||
continue
|
||
return f"alias:{left['suffix']}", left, right
|
||
return None
|
||
|
||
|
||
def _number_sort_key(number):
|
||
"""Разбить номер на текстовые и числовые части для сортировки ДС по номеру."""
|
||
return tuple((0, int(part)) if part.isdigit() else (1, part.casefold())
|
||
for part in re.split(r"(\d+)", str(number or "")))
|
||
|
||
|
||
def _dossier_exception(batch_id, document_id, title, source_quote):
|
||
"""Создать устойчивую запись исключения, не дублируя её при пересборке batch."""
|
||
existing = db_exceptions.list_exceptions(batch_id=batch_id)
|
||
if any(
|
||
item.get("document_id") == document_id
|
||
and item.get("stage") == "dossier_linking"
|
||
and item.get("title") == title
|
||
for item in existing
|
||
):
|
||
return
|
||
db_exceptions.add_exception(
|
||
batch_id=batch_id, dossier_id=None, document_id=document_id,
|
||
service_item_id=None, severity="AMBIGUOUS", stage="dossier_linking",
|
||
title=title, source_quote=source_quote,
|
||
)
|
||
|
||
|
||
def build_dossiers_for_batch(batch_id):
|
||
"""Atomically rebuild the dossier graph, retaining the old graph on failure."""
|
||
with transaction():
|
||
return _build_dossiers_for_batch(batch_id)
|
||
|
||
|
||
def _build_dossiers_for_batch(batch_id):
|
||
"""Построить и сохранить dossier graph из паспортов документов одного batch.
|
||
|
||
Полные номера и ссылки образуют рёбра только при совместимом контрагенте.
|
||
Компоненты дополнительно защищены от транзитивного слияния двух разных ИНН;
|
||
документ, который остаётся кандидатом сразу для нескольких таких компонент,
|
||
исключается из автоматических связей и передаётся в очередь исключений.
|
||
"""
|
||
documents = db_docs.list_by_batch(batch_id)
|
||
passports = db_extractions.list_by_batch(batch_id)
|
||
passport_by_id = {row["document_id"]: row for row in passports}
|
||
records = []
|
||
for document in documents:
|
||
passport = passport_by_id.get(document["id"])
|
||
if not passport:
|
||
passport = {"document_id": document["id"], "doc_class": "unknown"}
|
||
if passport.get("doc_class") == "ignored_accounting":
|
||
continue
|
||
counterparty = {
|
||
"name": passport.get("counterparty_name"),
|
||
"inn": passport.get("counterparty_inn"),
|
||
}
|
||
records.append({
|
||
"document": document,
|
||
"passport": passport,
|
||
"counterparty": counterparty,
|
||
"keys": _passport_keys(passport),
|
||
})
|
||
|
||
parent = list(range(len(records)))
|
||
component_inns = []
|
||
for record in records:
|
||
counterparty_id = canonicalize_counterparty(
|
||
record["counterparty"].get("name"), record["counterparty"].get("inn"),
|
||
)
|
||
component_inns.append({counterparty_id["inn"]} if counterparty_id["inn"] else set())
|
||
|
||
def find(index):
|
||
while parent[index] != index:
|
||
parent[index] = parent[parent[index]]
|
||
index = parent[index]
|
||
return index
|
||
|
||
candidates = [[] for _ in records]
|
||
for left_index, left in enumerate(records):
|
||
for right_index in range(left_index + 1, len(records)):
|
||
right = records[right_index]
|
||
match = _shared_contract_match(left["keys"], right["keys"])
|
||
if not match or not _counterparties_compatible(left["counterparty"], right["counterparty"]):
|
||
continue
|
||
shared_key, left_number, right_number = match
|
||
candidates[left_index].append((right_index, shared_key, left_number["raw"]))
|
||
candidates[right_index].append((left_index, shared_key, right_number["raw"]))
|
||
left_root, right_root = find(left_index), find(right_index)
|
||
if left_root == right_root:
|
||
continue
|
||
merged_inns = component_inns[left_root] | component_inns[right_root]
|
||
if len(merged_inns) > 1:
|
||
continue
|
||
parent[right_root] = left_root
|
||
component_inns[left_root] = merged_inns
|
||
|
||
ambiguous_indices = set()
|
||
for index, edges in enumerate(candidates):
|
||
key_targets = {}
|
||
for other, shared_key, _raw_number in edges:
|
||
key_targets.setdefault(shared_key, set()).add(find(other))
|
||
candidate_roots = set().union(*key_targets.values()) if key_targets else set()
|
||
if len(candidate_roots) > 1:
|
||
ambiguous_indices.add(index)
|
||
matching_numbers = sorted({
|
||
raw_number for _other, _key, raw_number in edges
|
||
})
|
||
_dossier_exception(
|
||
batch_id, records[index]["document"]["id"],
|
||
"AMBIGUOUS_DOSSIER_LINK", ", ".join(matching_numbers),
|
||
)
|
||
|
||
component_members = {}
|
||
for index, record in enumerate(records):
|
||
if index in ambiguous_indices:
|
||
continue
|
||
if not record["keys"]:
|
||
_dossier_exception(
|
||
batch_id, record["document"]["id"], "NO_RELIABLE_CONTRACT_NUMBER",
|
||
record["document"].get("filename") or "",
|
||
)
|
||
continue
|
||
component_members.setdefault(find(index), []).append(record)
|
||
|
||
db_dossiers.clear_batch_dossiers(batch_id)
|
||
dossier_results = []
|
||
for members in component_members.values():
|
||
frames = [r for r in members if r["passport"].get("doc_class") == "contract_frame"]
|
||
specifications = [r for r in members if r["passport"].get("doc_class") == "specification"]
|
||
specifications.sort(key=lambda r: (
|
||
r["passport"].get("own_date") or r["document"].get("doc_date") or "9999",
|
||
r["document"].get("id", ""),
|
||
))
|
||
identifiers = []
|
||
for record in members:
|
||
for key, value in record["keys"].items():
|
||
identifiers.append((key, value["raw"], value["source"]))
|
||
canonical_candidates = [
|
||
r["passport"].get("own_number") for r in frames
|
||
if _is_contract_number(r["passport"].get("own_number"))
|
||
]
|
||
canonical_candidates.extend(
|
||
reference.get("number")
|
||
for record in [*frames, *specifications]
|
||
for reference in (record["passport"].get("referenced_numbers") or [])
|
||
if isinstance(reference, dict) and _is_contract_number(reference.get("number"))
|
||
)
|
||
canonical_candidates.extend(
|
||
r["passport"].get("own_number") for r in specifications
|
||
if _is_contract_number(r["passport"].get("own_number"))
|
||
)
|
||
canonical_candidates.extend(
|
||
raw for _key, raw, _source in identifiers if _is_contract_number(raw)
|
||
)
|
||
canonical_number = canonical_candidates[0] if canonical_candidates else ""
|
||
canonical_key = canonicalize_contract_number(canonical_number)
|
||
aliases = list(dict.fromkeys(
|
||
raw for key, raw, _source in identifiers
|
||
if key != canonical_key and raw != canonical_number
|
||
))
|
||
party_candidates = [
|
||
record for record in members
|
||
if record["counterparty"].get("inn") or record["counterparty"].get("name")
|
||
]
|
||
party_rank = {"contract_frame": 0, "supplement": 1, "specification": 2}
|
||
party_candidates.sort(key=lambda record: party_rank.get(
|
||
record["passport"].get("doc_class"), 3,
|
||
))
|
||
counterparty_record = party_candidates[0]["counterparty"] if party_candidates else {}
|
||
dossier = db_dossiers.create_dossier(
|
||
batch_id=batch_id,
|
||
canonical_number=canonical_number,
|
||
alias_numbers=aliases,
|
||
counterparty_name=counterparty_record.get("name") or "",
|
||
counterparty_inn=counterparty_record.get("inn") or "",
|
||
has_frame_contract=bool(frames),
|
||
)
|
||
base_spec_id = specifications[0]["document"]["id"] if specifications else None
|
||
members.sort(key=lambda r: _dossier_order_key(r, base_spec_id))
|
||
linked_documents = []
|
||
for sequence, record in enumerate(members, start=1):
|
||
document = record["document"]
|
||
passport = record["passport"]
|
||
doc_class = passport.get("doc_class")
|
||
if doc_class == "contract_frame":
|
||
role = "frame_contract"
|
||
elif doc_class == "specification" and document["id"] == base_spec_id:
|
||
role = "base_spec"
|
||
elif doc_class == "specification":
|
||
role = "specification"
|
||
elif doc_class == "supplement":
|
||
role = "supplement"
|
||
else:
|
||
role = "document"
|
||
reason = _link_reason(record, canonical_key, canonical_number)
|
||
linked_party = canonicalize_counterparty(
|
||
record["counterparty"].get("name"), record["counterparty"].get("inn"),
|
||
)
|
||
selected_party = canonicalize_counterparty(
|
||
counterparty_record.get("name"), counterparty_record.get("inn"),
|
||
)
|
||
if linked_party["name"] and selected_party["name"] and linked_party["name"] != selected_party["name"]:
|
||
reason += (
|
||
f"; расхождение контрагента: {record['counterparty']['name']}"
|
||
f" (в досье: {counterparty_record['name']})"
|
||
)
|
||
link = db_dossiers.link_document(
|
||
dossier_id=dossier["id"], document_id=document["id"], role=role,
|
||
link_status="CONFIRMED", link_reason=reason, seq_order=sequence,
|
||
)
|
||
linked_documents.append({**document, "role": role, "link_reason": reason, "link": link})
|
||
dossier_results.append({**dossier, "documents": linked_documents})
|
||
|
||
return {
|
||
"ok": True,
|
||
"batch_id": batch_id,
|
||
"total_docs": len(documents),
|
||
"dossiers": dossier_results,
|
||
"unlinked": sorted(records[i]["document"]["id"] for i in ambiguous_indices)
|
||
+ [r["document"]["id"] for r in records if not r["keys"]],
|
||
"ambiguous": len(ambiguous_indices),
|
||
}
|
||
|
||
|
||
def _dossier_order_key(record, base_spec_id):
|
||
"""Упорядочить документы так, чтобы рамка и базовая спека шли первыми."""
|
||
document = record["document"]
|
||
passport = record["passport"]
|
||
doc_class = passport.get("doc_class")
|
||
if doc_class == "contract_frame":
|
||
rank = 0
|
||
elif doc_class == "specification" and document["id"] == base_spec_id:
|
||
rank = 1
|
||
elif doc_class == "specification":
|
||
rank = 2
|
||
elif doc_class == "supplement":
|
||
rank = 3
|
||
else:
|
||
rank = 4
|
||
if doc_class == "supplement":
|
||
secondary = (_number_sort_key(passport.get("own_number")),
|
||
passport.get("own_date") or document.get("doc_date") or "9999")
|
||
else:
|
||
secondary = (passport.get("own_date") or document.get("doc_date") or "9999",)
|
||
return rank, secondary, document.get("id", "")
|
||
|
||
|
||
def _link_reason(record, canonical_key, canonical_number):
|
||
"""Сформировать аудируемое основание связи из совпавшей ссылки документа."""
|
||
for reference in record["passport"].get("referenced_numbers") or []:
|
||
if not isinstance(reference, dict) or not reference.get("number"):
|
||
continue
|
||
if canonicalize_contract_number(reference["number"]) == canonical_key:
|
||
source = reference.get("source")
|
||
if source in {"header", "preamble", "appendix"}:
|
||
return f"Связано по номеру {reference['number']} из {source}"
|
||
return f"Связано по номеру {reference['number']}"
|
||
own_number = record["passport"].get("own_number")
|
||
if own_number and canonicalize_contract_number(own_number) == canonical_key:
|
||
return f"Собственный номер документа совпадает с {canonical_number}"
|
||
return f"Связано в dossier по номеру {canonical_number}"
|
||
|
||
|
||
def group_documents(batch_id):
|
||
"""
|
||
Сгруппировать классифицированные документы по контрактам.
|
||
|
||
Алгоритм:
|
||
1. Отделить contract от supplement/specification
|
||
2. Каждый contract → якорь группы
|
||
3. Для каждого supplement: найти contract по parent_number (нормализованный)
|
||
4. Оставшиеся supplement/spec → виртуальные группы по parent_number/own_number
|
||
5. Совсем без номеров → группа "__unresolved__"
|
||
6. Внутри группы сортировка по doc_date
|
||
|
||
Возвращает {ok, groups: [{contract_number, counterparty, documents: [...]}], total_docs}.
|
||
"""
|
||
docs = db_docs.list_by_batch(batch_id)
|
||
classified = [d for d in docs if d.get("classify_status") == "classified"]
|
||
|
||
# Separate contracts and supplements
|
||
contracts_list = []
|
||
supplements_list = []
|
||
|
||
for d in classified:
|
||
if d.get("doc_type") == "contract":
|
||
contracts_list.append(d)
|
||
else:
|
||
supplements_list.append(d)
|
||
|
||
groups = []
|
||
|
||
# Each contract becomes a group
|
||
for c in contracts_list:
|
||
group = {
|
||
"contract_number": c.get("own_number") or c.get("filename", ""),
|
||
"counterparty": c.get("counterparty") or "",
|
||
"documents": [c],
|
||
}
|
||
# Find supplements matching this contract
|
||
c_norm = normalize_number(c.get("own_number"))
|
||
for s in supplements_list:
|
||
parent = normalize_number(s.get("parent_number") or "")
|
||
own = normalize_number(s.get("own_number") or "")
|
||
if parent == c_norm or own == c_norm:
|
||
if s not in group["documents"]:
|
||
group["documents"].append(s)
|
||
|
||
# Sort by date
|
||
group["documents"].sort(key=lambda x: x.get("doc_date") or "")
|
||
|
||
# Remove matched supplements from the pool
|
||
for s in group["documents"]:
|
||
if s in supplements_list:
|
||
supplements_list.remove(s)
|
||
|
||
groups.append(group)
|
||
|
||
# ── Virtual groups: unmatched supplements grouped by number ──────────
|
||
# Group remaining supplements/specs by normalized parent_number (priority) or own_number
|
||
virtual = {}
|
||
for s in supplements_list:
|
||
num = normalize_number(s.get("parent_number") or s.get("own_number") or "")
|
||
if not num:
|
||
continue # no number → stays in supplements_list for unresolved
|
||
if num not in virtual:
|
||
# Use counterparty from first doc in group
|
||
cp = s.get("counterparty") or ""
|
||
virtual[num] = {
|
||
"contract_number": s.get("parent_number") or s.get("own_number") or "?",
|
||
"counterparty": cp,
|
||
"documents": [],
|
||
}
|
||
virtual[num]["documents"].append(s)
|
||
# Update counterparty if current doc has a better one
|
||
if not virtual[num]["counterparty"] and s.get("counterparty"):
|
||
virtual[num]["counterparty"] = s.get("counterparty")
|
||
|
||
for vnum, vgroup in virtual.items():
|
||
vgroup["documents"].sort(key=lambda x: x.get("doc_date") or "")
|
||
groups.append(vgroup)
|
||
# Remove grouped docs from supplements_list
|
||
for s in vgroup["documents"]:
|
||
supplements_list.remove(s)
|
||
|
||
# ── Unresolved: everything left (no number, failed, pending, other) ──
|
||
unmatched = [s for s in supplements_list] # remaining after virtual grouping
|
||
unmatched += [d for d in docs if d.get("classify_status") != "classified"]
|
||
|
||
if unmatched:
|
||
groups.append({
|
||
"contract_number": "__unresolved__",
|
||
"counterparty": "",
|
||
"documents": unmatched,
|
||
})
|
||
|
||
return {"ok": True, "groups": groups, "total_docs": len(docs)}
|
||
|
||
|
||
def apply_groups(batch_id, groups_data):
|
||
"""
|
||
Применить подтверждённые группы: создать contracts + supplements.
|
||
|
||
Вызывается из POST /api/apply-groups.
|
||
Для каждой группы (кроме __unresolved__):
|
||
1. Создать запись в contracts (number, client)
|
||
2. Для каждого документа создать supplement (type='initial' для первого, 'additional' для остальных)
|
||
3. Порядок supplements соответствует порядку документов в группе (сортировка по дате уже сделана)
|
||
|
||
Возвращает {ok, created: количество созданных supplements}.
|
||
"""
|
||
created = 0
|
||
contract_ids = []
|
||
for g in groups_data:
|
||
contract_number = g.get("contract_number", "")
|
||
if contract_number == "__unresolved__":
|
||
continue
|
||
counterparty = g.get("counterparty", "")
|
||
docs = g.get("documents", [])
|
||
|
||
try:
|
||
c = db_contracts.insert(contract_number, counterparty)
|
||
contract_id = c["id"]
|
||
contract_ids.append(contract_id)
|
||
|
||
for i, d in enumerate(docs):
|
||
doc_id = d.get("id")
|
||
if not doc_id:
|
||
continue
|
||
supp_type = "initial" if i == 0 else "additional"
|
||
db_supplements.insert(contract_id, doc_id, supp_type)
|
||
created += 1
|
||
except Exception as e:
|
||
return {"ok": False, "error": f"apply_groups failed at {contract_number}: {e}"}
|
||
|
||
return {"ok": True, "created": created, "contract_ids": contract_ids}
|