381 lines
18 KiB
Python
381 lines
18 KiB
Python
"""Сверка внешней CRM/1С-выгрузки с утверждёнными интервалами dossier.
|
||
|
||
Сверка выполняется по исходным суммам и датам на Decimal/date. Отсутствие
|
||
известного SKU не блокирует проверку договора; неизвестный VAT rate, напротив,
|
||
не позволяет объявить цены совпавшими, если стороны указали разные VAT modes.
|
||
"""
|
||
import re
|
||
from datetime import date, datetime, timedelta
|
||
from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
|
||
|
||
from db import dossiers as db_dossiers
|
||
from db import exceptions as db_exceptions
|
||
from db import timeline as db_timeline
|
||
from services.catalog_matcher import match_catalog_sku
|
||
from services.metrics import normalize_date
|
||
from services.service_identity import analyze_service_row, match_service_item
|
||
|
||
|
||
MONEY_TOLERANCE = Decimal("0.01")
|
||
|
||
|
||
def _decimal(value):
|
||
"""Преобразовать внешнее или договорное число в конечный Decimal."""
|
||
if value is None or value == "" or isinstance(value, bool):
|
||
return None
|
||
try:
|
||
result = Decimal(str(value).replace("\xa0", "").replace(" ", "").replace(",", "."))
|
||
except (InvalidOperation, ValueError, TypeError):
|
||
return None
|
||
return result if result.is_finite() else None
|
||
|
||
|
||
def _date(value):
|
||
"""Разобрать поддерживаемую ISO/DD.MM.YYYY дату без принятия мусорных значений."""
|
||
if not value:
|
||
return None
|
||
try:
|
||
return datetime.strptime(normalize_date(str(value).strip()), "%Y-%m-%d").date()
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _window(target_date_or_period):
|
||
"""Нормализовать точечную дату или полуоткрытый период [start, end)."""
|
||
if target_date_or_period is None:
|
||
point = date.today()
|
||
return {"kind": "point", "date": point, "start": point, "end": point + timedelta(days=1)}
|
||
if isinstance(target_date_or_period, (tuple, list)) and len(target_date_or_period) == 2:
|
||
start, end = map(_date, target_date_or_period)
|
||
elif isinstance(target_date_or_period, dict):
|
||
start = _date(target_date_or_period.get("start") or target_date_or_period.get("period_start"))
|
||
end = _date(target_date_or_period.get("end") or target_date_or_period.get("period_end"))
|
||
else:
|
||
point = _date(target_date_or_period)
|
||
if point is None:
|
||
raise ValueError("target date must be YYYY-MM-DD or DD.MM.YYYY")
|
||
return {"kind": "point", "date": point, "start": point, "end": point + timedelta(days=1)}
|
||
if start is None or end is None or end <= start:
|
||
raise ValueError("target period must have valid start < end")
|
||
return {"kind": "period", "date": None, "start": start, "end": end}
|
||
|
||
|
||
def _interval_overlaps(interval, window):
|
||
"""Проверить активность интервала на дату или пересечение с period window."""
|
||
start = _date(interval.get("valid_from"))
|
||
end = _date(interval.get("valid_to"))
|
||
if start is None:
|
||
return False
|
||
return start < window["end"] and (end is None or end > window["start"])
|
||
|
||
|
||
def _external_end(row):
|
||
"""Согласовать inclusive date_end CRM со stored exclusive valid_to."""
|
||
exclusive = _date(row.get("date_end_exclusive"))
|
||
if exclusive:
|
||
return exclusive
|
||
inclusive = _date(row.get("date_end") or row.get("valid_to"))
|
||
if not inclusive:
|
||
return None
|
||
return inclusive + timedelta(days=1) if row.get("date_end_inclusive", True) else inclusive
|
||
|
||
|
||
def _amount(row, sum_key="sum", price_key="price", qty_key="qty"):
|
||
"""Получить сумму напрямую либо вычислить её как точное price × qty."""
|
||
total = _decimal(row.get(sum_key))
|
||
if total is not None:
|
||
return total
|
||
price, qty = _decimal(row.get(price_key)), _decimal(row.get(qty_key))
|
||
return price * qty if price is not None and qty is not None else None
|
||
|
||
|
||
def _vat_mode(value):
|
||
"""Свести известные подписи режима НДС к gross/net/unknown."""
|
||
mode = str(value or "unknown").strip().lower()
|
||
if mode in {"with_vat", "vat_included", "inclusive", "gross", "с ндс", "включен"}:
|
||
return "gross"
|
||
if mode in {"without_vat", "vat_excluded", "exclusive", "net", "без ндс", "не включен"}:
|
||
return "net"
|
||
return "unknown"
|
||
|
||
|
||
def _vat_rate(row):
|
||
"""Прочитать только явно переданную ставку НДС; не выводить её по календарю."""
|
||
value = _decimal(row.get("vat_rate"))
|
||
if value is None:
|
||
return None
|
||
if value > 1:
|
||
value /= Decimal("100")
|
||
if value < 0 or value > 1:
|
||
return None
|
||
return value
|
||
|
||
|
||
def _to_gross(amount, mode, rate):
|
||
"""Нормализовать amount к gross, округляя по правилам денежных копеек."""
|
||
normalized_mode = _vat_mode(mode)
|
||
if amount is None:
|
||
return None
|
||
if normalized_mode == "gross":
|
||
return amount.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)
|
||
if normalized_mode == "net" and rate is not None:
|
||
return (amount * (Decimal("1") + rate)).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)
|
||
return None
|
||
|
||
|
||
def _vat_comparable(contract, external):
|
||
"""Разрешить денежную сверку при одинаковом режиме или двух известных ставках."""
|
||
contract_mode, external_mode = _vat_mode(contract.get("vat_mode")), _vat_mode(external.get("vat_mode"))
|
||
if contract_mode == external_mode and contract_mode != "unknown":
|
||
return True
|
||
if contract_mode == "unknown" and external_mode == "unknown":
|
||
return True
|
||
return _vat_rate(contract) is not None and _vat_rate(external) is not None
|
||
|
||
|
||
def _money_comparison(contract, external):
|
||
"""Сравнить amount, price и qty с точным VAT normalization."""
|
||
if not _vat_comparable(contract, external):
|
||
return {"comparable": False, "matches": False, "reason": "VAT mode/rate is not comparable"}
|
||
contract_rate, external_rate = _vat_rate(contract), _vat_rate(external)
|
||
contract_amount = _to_gross(_amount(contract), contract.get("vat_mode"), contract_rate)
|
||
external_amount = _to_gross(_amount(external), external.get("vat_mode"), external_rate)
|
||
if contract_amount is None or external_amount is None:
|
||
return {"comparable": False, "matches": False, "reason": "amount is missing"}
|
||
contract_qty, external_qty = _decimal(contract.get("qty")), _decimal(external.get("qty"))
|
||
if contract_qty is not None and external_qty is not None and contract_qty != external_qty:
|
||
return {"comparable": True, "matches": False, "reason": "quantity differs"}
|
||
contract_price = _to_gross(_decimal(contract.get("price")), contract.get("vat_mode"), contract_rate)
|
||
external_price = _to_gross(_decimal(external.get("price")), external.get("vat_mode"), external_rate)
|
||
if contract_price is not None and external_price is not None:
|
||
if abs(contract_price - external_price) > MONEY_TOLERANCE:
|
||
return {"comparable": True, "matches": False, "reason": "unit price differs"}
|
||
if abs(contract_amount - external_amount) > MONEY_TOLERANCE:
|
||
return {"comparable": True, "matches": False, "reason": "sum differs"}
|
||
return {"comparable": True, "matches": True, "reason": ""}
|
||
|
||
|
||
def _find_service(external, items, catalog_entries):
|
||
"""Сопоставить external row по явному ID, SKU или семейству/anchor."""
|
||
explicit_id = external.get("service_item_id")
|
||
if explicit_id:
|
||
exact = [item for item in items if item.get("id") == explicit_id]
|
||
if exact:
|
||
return {"status": "MATCHED", "service_item": exact[0], "candidates": exact}
|
||
external_sku = re.sub(r"[^0-9A-ZА-ЯЁ-]", "", str(external.get("sku") or "").upper())
|
||
if external_sku:
|
||
sku_matches = [
|
||
item for item in items
|
||
if re.sub(r"[^0-9A-ZА-ЯЁ-]", "", str(item.get("catalog_sku") or "").upper()) == external_sku
|
||
]
|
||
if len(sku_matches) == 1:
|
||
return {"status": "MATCHED", "service_item": sku_matches[0], "candidates": sku_matches}
|
||
raw_name = external.get("name") or external.get("service_name") or ""
|
||
if external_sku and catalog_entries:
|
||
sku_candidates = []
|
||
for item in items:
|
||
catalog_result = match_catalog_sku(
|
||
{**item, "charge_type": external.get("charge_type")}, catalog_entries,
|
||
)
|
||
if catalog_result.get("sku") and str(catalog_result["sku"]).upper() == external_sku:
|
||
sku_candidates.append(item)
|
||
if len(sku_candidates) == 1:
|
||
return {"status": "MATCHED", "service_item": sku_candidates[0], "candidates": sku_candidates}
|
||
if sku_candidates:
|
||
return {"status": "AMBIGUOUS", "service_item": None, "candidates": sku_candidates}
|
||
identity = analyze_service_row(raw_name, external.get("charge_type"))
|
||
return match_service_item(identity, items)
|
||
|
||
|
||
def _date_comparison(interval, external):
|
||
"""Сверить точные границы external row с интервалом договора."""
|
||
contract_start = _date(interval.get("valid_from"))
|
||
external_start = _date(external.get("date_start") or external.get("valid_from"))
|
||
contract_end = _date(interval.get("valid_to"))
|
||
external_end = _external_end(external)
|
||
if external_start != contract_start or external_end != contract_end:
|
||
return {
|
||
"matches": False,
|
||
"contract_start": contract_start.isoformat() if contract_start else None,
|
||
"external_start": external_start.isoformat() if external_start else None,
|
||
"contract_end_exclusive": contract_end.isoformat() if contract_end else None,
|
||
"external_end_exclusive": external_end.isoformat() if external_end else None,
|
||
}
|
||
return {"matches": True}
|
||
|
||
|
||
def _finding(status, external=None, service_item=None, interval=None, reason="", **details):
|
||
"""Собрать отчётный finding только из JSON-совместимых значений."""
|
||
return {
|
||
"status": status,
|
||
"external_row": _json_safe(external),
|
||
"service_item_id": service_item.get("id") if service_item else None,
|
||
"service_name": (interval or {}).get("raw_name") or (service_item or {}).get("canonical_name"),
|
||
"interval_id": (interval or {}).get("id"),
|
||
"reason": reason,
|
||
**details,
|
||
}
|
||
|
||
|
||
def _json_safe(value):
|
||
"""Сериализовать Decimal/date в точные строки для Flask JSON response."""
|
||
if isinstance(value, Decimal):
|
||
return format(value, "f")
|
||
if isinstance(value, (date, datetime)):
|
||
return value.isoformat()
|
||
if isinstance(value, dict):
|
||
return {key: _json_safe(item) for key, item in value.items()}
|
||
if isinstance(value, (list, tuple)):
|
||
return [_json_safe(item) for item in value]
|
||
return value
|
||
|
||
|
||
def reconcile_dossier(dossier_id, external_rows, target_date_or_period=None,
|
||
catalog_entries=None):
|
||
"""Сверить CRM/1С строки с активными обязательствами dossier.
|
||
|
||
Входные даты `date_end` считаются включительными; `date_end_exclusive` можно
|
||
передать для явной полуоткрытой границы. Неоднозначные service matches и
|
||
открытые исключения получают NEEDS_HUMAN_DECISION. SKU UNKNOWN не меняет
|
||
результат сверки дат, количеств и денежных сумм.
|
||
"""
|
||
dossier = db_dossiers.get_dossier(dossier_id)
|
||
if dossier is None:
|
||
return {"ok": False, "error": "dossier not found", "dossier_id": dossier_id}
|
||
window = _window(target_date_or_period)
|
||
items = db_timeline.list_service_items(dossier_id)
|
||
intervals = [
|
||
interval for interval in db_timeline.list_timeline(dossier_id)
|
||
if _interval_overlaps(interval, window)
|
||
]
|
||
open_exceptions = db_exceptions.list_exceptions(dossier_id=dossier_id, status="OPEN")
|
||
exceptions_by_item = {}
|
||
for issue in open_exceptions:
|
||
service_item_id = issue.get("service_item_id")
|
||
if service_item_id:
|
||
exceptions_by_item.setdefault(service_item_id, []).append(issue)
|
||
|
||
findings = []
|
||
observed_items = set()
|
||
uncertain_items = set()
|
||
for row_index, external in enumerate(external_rows or [], start=1):
|
||
if not isinstance(external, dict):
|
||
findings.append(_finding("NEEDS_HUMAN_DECISION", reason="external row must be an object", row_index=row_index))
|
||
continue
|
||
match = _find_service(external, items, catalog_entries or [])
|
||
if match["status"] == "AMBIGUOUS":
|
||
candidate_ids = [item["id"] for item in match["candidates"]]
|
||
uncertain_items.update(candidate_ids)
|
||
findings.append(_finding(
|
||
"NEEDS_HUMAN_DECISION", external=external, reason="service match is ambiguous",
|
||
candidate_service_item_ids=candidate_ids,
|
||
))
|
||
continue
|
||
service_item = match.get("service_item")
|
||
if service_item is None:
|
||
findings.append(_finding(
|
||
"UNCONTRACTED_BILLING", external=external,
|
||
reason="no matching service item exists in dossier",
|
||
))
|
||
continue
|
||
observed_items.add(service_item["id"])
|
||
catalog_status = service_item.get("sku_status") or "UNKNOWN"
|
||
if catalog_entries:
|
||
catalog_result = match_catalog_sku(
|
||
{**service_item, "charge_type": external.get("charge_type")},
|
||
catalog_entries,
|
||
)
|
||
catalog_status = catalog_result["status"]
|
||
if service_item["id"] in exceptions_by_item:
|
||
findings.append(_finding(
|
||
"NEEDS_HUMAN_DECISION", external=external, service_item=service_item,
|
||
reason="service item has open dossier exceptions",
|
||
sku_status=catalog_status,
|
||
exception_ids=[issue["id"] for issue in exceptions_by_item[service_item["id"]]],
|
||
))
|
||
continue
|
||
|
||
matching_intervals = [
|
||
interval for interval in intervals
|
||
if interval.get("service_item_id") == service_item["id"]
|
||
and (not external.get("charge_type") or interval.get("charge_type") == external.get("charge_type"))
|
||
]
|
||
if not matching_intervals:
|
||
findings.append(_finding(
|
||
"UNCONTRACTED_BILLING", external=external, service_item=service_item,
|
||
reason="service item has no contractual interval in the target window",
|
||
sku_status=catalog_status,
|
||
))
|
||
continue
|
||
if len(matching_intervals) > 1:
|
||
exact = [interval for interval in matching_intervals if _date_comparison(interval, external)["matches"]]
|
||
if len(exact) == 1:
|
||
matching_intervals = exact
|
||
else:
|
||
uncertain_items.add(service_item["id"])
|
||
findings.append(_finding(
|
||
"NEEDS_HUMAN_DECISION", external=external, service_item=service_item,
|
||
reason="external period overlaps multiple contractual intervals",
|
||
sku_status=catalog_status,
|
||
candidate_interval_ids=[interval["id"] for interval in matching_intervals],
|
||
))
|
||
continue
|
||
|
||
interval = matching_intervals[0]
|
||
dates = _date_comparison(interval, external)
|
||
if not dates["matches"]:
|
||
findings.append(_finding(
|
||
"DATE_MISMATCH", external=external, service_item=service_item,
|
||
interval=interval, reason="date range differs", sku_status=catalog_status, **dates,
|
||
))
|
||
continue
|
||
money = _money_comparison(interval, external)
|
||
if not money["comparable"]:
|
||
findings.append(_finding(
|
||
"NEEDS_HUMAN_DECISION", external=external, service_item=service_item,
|
||
interval=interval, reason=money["reason"], sku_status=catalog_status,
|
||
))
|
||
elif not money["matches"]:
|
||
findings.append(_finding(
|
||
"PRICE_OR_QTY_MISMATCH", external=external, service_item=service_item,
|
||
interval=interval, reason=money["reason"], sku_status=catalog_status,
|
||
))
|
||
else:
|
||
findings.append(_finding(
|
||
"MATCH", external=external, service_item=service_item, interval=interval,
|
||
reason="dates, quantity and amount match", sku_status=catalog_status,
|
||
))
|
||
|
||
for interval in intervals:
|
||
service_item_id = interval.get("service_item_id")
|
||
if service_item_id in observed_items or service_item_id in uncertain_items:
|
||
continue
|
||
if interval.get("charge_type") == "payg":
|
||
continue
|
||
service_item = next((item for item in items if item.get("id") == service_item_id), None)
|
||
if service_item_id in exceptions_by_item:
|
||
findings.append(_finding(
|
||
"NEEDS_HUMAN_DECISION", service_item=service_item, interval=interval,
|
||
reason="service item has open dossier exceptions",
|
||
exception_ids=[issue["id"] for issue in exceptions_by_item[service_item_id]],
|
||
))
|
||
else:
|
||
findings.append(_finding(
|
||
"UNBILLED_SERVICE", service_item=service_item, interval=interval,
|
||
reason="contractual service is active but absent from external rows",
|
||
))
|
||
|
||
summary = {}
|
||
for finding in findings:
|
||
summary[finding["status"]] = summary.get(finding["status"], 0) + 1
|
||
return {
|
||
"ok": True,
|
||
"dossier_id": dossier_id,
|
||
"target": {
|
||
"kind": window["kind"],
|
||
"start": window["start"].isoformat(),
|
||
"end_exclusive": window["end"].isoformat(),
|
||
},
|
||
"summary": summary,
|
||
"findings": findings,
|
||
} |