@@ -0,0 +1,381 @@
""" Сверка внешней CRM/1С-выгрузки с утверждёнными интервалами dossier.
Сверка выполняется по исходным суммам и датам на Decimal/date. Отсутствие
известного SKU не блокирует проверку договора; неизвестный VAT rate, напротив,
не позволяет объявить цены совпавшими, если стороны указали разные VAT modes.
"""
import re
from datetime import date , datetime , timedelta
from decimal import Decimal , InvalidOperation , ROUND_HALF_UP
from db import dossiers as db_dossiers
from db import exceptions as db_exceptions
from db import timeline as db_timeline
from services . catalog_matcher import match_catalog_sku
from services . metrics import normalize_date
from services . service_identity import analyze_service_row , match_service_item
MONEY_TOLERANCE = Decimal ( " 0.01 " )
def _decimal ( value ) :
""" Преобразовать внешнее или договорное число в конечный Decimal. """
if value is None or value == " " or isinstance ( value , bool ) :
return None
try :
result = Decimal ( str ( value ) . replace ( " \xa0 " , " " ) . replace ( " " , " " ) . replace ( " , " , " . " ) )
except ( InvalidOperation , ValueError , TypeError ) :
return None
return result if result . is_finite ( ) else None
def _date ( value ) :
""" Разобрать поддерживаемую ISO/DD.MM.YYYY дату без принятия мусорных значений. """
if not value :
return None
try :
return datetime . strptime ( normalize_date ( str ( value ) . strip ( ) ) , " % Y- % m- %d " ) . date ( )
except ( TypeError , ValueError ) :
return None
def _window ( target_date_or_period ) :
""" Нормализовать точечную дату или полуоткрытый период [start, end). """
if target_date_or_period is None :
point = date . today ( )
return { " kind " : " point " , " date " : point , " start " : point , " end " : point + timedelta ( days = 1 ) }
if isinstance ( target_date_or_period , ( tuple , list ) ) and len ( target_date_or_period ) == 2 :
start , end = map ( _date , target_date_or_period )
elif isinstance ( target_date_or_period , dict ) :
start = _date ( target_date_or_period . get ( " start " ) or target_date_or_period . get ( " period_start " ) )
end = _date ( target_date_or_period . get ( " end " ) or target_date_or_period . get ( " period_end " ) )
else :
point = _date ( target_date_or_period )
if point is None :
raise ValueError ( " target date must be YYYY-MM-DD or DD.MM.YYYY " )
return { " kind " : " point " , " date " : point , " start " : point , " end " : point + timedelta ( days = 1 ) }
if start is None or end is None or end < = start :
raise ValueError ( " target period must have valid start < end " )
return { " kind " : " period " , " date " : None , " start " : start , " end " : end }
def _interval_overlaps ( interval , window ) :
""" Проверить активность интервала на дату или пересечение с period window. """
start = _date ( interval . get ( " valid_from " ) )
end = _date ( interval . get ( " valid_to " ) )
if start is None :
return False
return start < window [ " end " ] and ( end is None or end > window [ " start " ] )
def _external_end ( row ) :
""" Согласовать inclusive date_end CRM со stored exclusive valid_to. """
exclusive = _date ( row . get ( " date_end_exclusive " ) )
if exclusive :
return exclusive
inclusive = _date ( row . get ( " date_end " ) or row . get ( " valid_to " ) )
if not inclusive :
return None
return inclusive + timedelta ( days = 1 ) if row . get ( " date_end_inclusive " , True ) else inclusive
def _amount ( row , sum_key = " sum " , price_key = " price " , qty_key = " qty " ) :
""" Получить сумму напрямую либо вычислить её как точное price × qty. """
total = _decimal ( row . get ( sum_key ) )
if total is not None :
return total
price , qty = _decimal ( row . get ( price_key ) ) , _decimal ( row . get ( qty_key ) )
return price * qty if price is not None and qty is not None else None
def _vat_mode ( value ) :
""" Свести известные подписи режима НДС к gross/net/unknown. """
mode = str ( value or " unknown " ) . strip ( ) . lower ( )
if mode in { " with_vat " , " vat_included " , " inclusive " , " gross " , " с " , " включен " } :
return " gross "
if mode in { " without_vat " , " vat_excluded " , " exclusive " , " net " , " без ндс " , " не включен " } :
return " net "
return " unknown "
def _vat_rate ( row ) :
""" Прочитать только явно переданную ставку НДС; не выводить её по календарю. """
value = _decimal ( row . get ( " vat_rate " ) )
if value is None :
return None
if value > 1 :
value / = Decimal ( " 100 " )
if value < 0 or value > 1 :
return None
return value
def _to_gross ( amount , mode , rate ) :
""" Нормализовать amount к gross, округляя по правилам денежных копеек. """
normalized_mode = _vat_mode ( mode )
if amount is None :
return None
if normalized_mode == " gross " :
return amount . quantize ( Decimal ( " 0.01 " ) , rounding = ROUND_HALF_UP )
if normalized_mode == " net " and rate is not None :
return ( amount * ( Decimal ( " 1 " ) + rate ) ) . quantize ( Decimal ( " 0.01 " ) , rounding = ROUND_HALF_UP )
return None
def _vat_comparable ( contract , external ) :
""" Разрешить денежную сверку при одинаковом режиме или двух известных ставках. """
contract_mode , external_mode = _vat_mode ( contract . get ( " vat_mode " ) ) , _vat_mode ( external . get ( " vat_mode " ) )
if contract_mode == external_mode and contract_mode != " unknown " :
return True
if contract_mode == " unknown " and external_mode == " unknown " :
return True
return _vat_rate ( contract ) is not None and _vat_rate ( external ) is not None
def _money_comparison ( contract , external ) :
""" Сравнить amount, price и qty с """
if not _vat_comparable ( contract , external ) :
return { " comparable " : False , " matches " : False , " reason " : " VAT mode/rate is not comparable " }
contract_rate , external_rate = _vat_rate ( contract ) , _vat_rate ( external )
contract_amount = _to_gross ( _amount ( contract ) , contract . get ( " vat_mode " ) , contract_rate )
external_amount = _to_gross ( _amount ( external ) , external . get ( " vat_mode " ) , external_rate )
if contract_amount is None or external_amount is None :
return { " comparable " : False , " matches " : False , " reason " : " amount is missing " }
contract_qty , external_qty = _decimal ( contract . get ( " qty " ) ) , _decimal ( external . get ( " qty " ) )
if contract_qty is not None and external_qty is not None and contract_qty != external_qty :
return { " comparable " : True , " matches " : False , " reason " : " quantity differs " }
contract_price = _to_gross ( _decimal ( contract . get ( " price " ) ) , contract . get ( " vat_mode " ) , contract_rate )
external_price = _to_gross ( _decimal ( external . get ( " price " ) ) , external . get ( " vat_mode " ) , external_rate )
if contract_price is not None and external_price is not None :
if abs ( contract_price - external_price ) > MONEY_TOLERANCE :
return { " comparable " : True , " matches " : False , " reason " : " unit price differs " }
if abs ( contract_amount - external_amount ) > MONEY_TOLERANCE :
return { " comparable " : True , " matches " : False , " reason " : " sum differs " }
return { " comparable " : True , " matches " : True , " reason " : " " }
def _find_service ( external , items , catalog_entries ) :
""" Сопоставить external row по явному ID, SKU или семейству/anchor. """
explicit_id = external . get ( " service_item_id " )
if explicit_id :
exact = [ item for item in items if item . get ( " id " ) == explicit_id ]
if exact :
return { " status " : " MATCHED " , " service_item " : exact [ 0 ] , " candidates " : exact }
external_sku = re . sub ( r " [^0-9A-ZА " , " " , str ( external . get ( " sku " ) or " " ) . upper ( ) )
if external_sku :
sku_matches = [
item for item in items
if re . sub ( r " [^0-9A-ZА " , " " , str ( item . get ( " catalog_sku " ) or " " ) . upper ( ) ) == external_sku
]
if len ( sku_matches ) == 1 :
return { " status " : " MATCHED " , " service_item " : sku_matches [ 0 ] , " candidates " : sku_matches }
raw_name = external . get ( " name " ) or external . get ( " service_name " ) or " "
if external_sku and catalog_entries :
sku_candidates = [ ]
for item in items :
catalog_result = match_catalog_sku (
{ * * item , " charge_type " : external . get ( " charge_type " ) } , catalog_entries ,
)
if catalog_result . get ( " sku " ) and str ( catalog_result [ " sku " ] ) . upper ( ) == external_sku :
sku_candidates . append ( item )
if len ( sku_candidates ) == 1 :
return { " status " : " MATCHED " , " service_item " : sku_candidates [ 0 ] , " candidates " : sku_candidates }
if sku_candidates :
return { " status " : " AMBIGUOUS " , " service_item " : None , " candidates " : sku_candidates }
identity = analyze_service_row ( raw_name , external . get ( " charge_type " ) )
return match_service_item ( identity , items )
def _date_comparison ( interval , external ) :
""" Сверить точные границы external row с интервалом договора. """
contract_start = _date ( interval . get ( " valid_from " ) )
external_start = _date ( external . get ( " date_start " ) or external . get ( " valid_from " ) )
contract_end = _date ( interval . get ( " valid_to " ) )
external_end = _external_end ( external )
if external_start != contract_start or external_end != contract_end :
return {
" matches " : False ,
" contract_start " : contract_start . isoformat ( ) if contract_start else None ,
" external_start " : external_start . isoformat ( ) if external_start else None ,
" contract_end_exclusive " : contract_end . isoformat ( ) if contract_end else None ,
" external_end_exclusive " : external_end . isoformat ( ) if external_end else None ,
}
return { " matches " : True }
def _finding ( status , external = None , service_item = None , interval = None , reason = " " , * * details ) :
""" Собрать отчётный finding только из JSON-совместимых значений. """
return {
" status " : status ,
" external_row " : _json_safe ( external ) ,
" service_item_id " : service_item . get ( " id " ) if service_item else None ,
" service_name " : ( interval or { } ) . get ( " raw_name " ) or ( service_item or { } ) . get ( " canonical_name " ) ,
" interval_id " : ( interval or { } ) . get ( " id " ) ,
" reason " : reason ,
* * details ,
}
def _json_safe ( value ) :
""" Сериализовать Decimal/date в точные строки для Flask JSON response. """
if isinstance ( value , Decimal ) :
return format ( value , " f " )
if isinstance ( value , ( date , datetime ) ) :
return value . isoformat ( )
if isinstance ( value , dict ) :
return { key : _json_safe ( item ) for key , item in value . items ( ) }
if isinstance ( value , ( list , tuple ) ) :
return [ _json_safe ( item ) for item in value ]
return value
def reconcile_dossier ( dossier_id , external_rows , target_date_or_period = None ,
catalog_entries = None ) :
""" Сверить CRM/1С строки с активными обязательствами dossier.
Входные даты `date_end` считаются включительными; `date_end_exclusive` можно
передать для явной полуоткрытой границы. Неоднозначные service matches и
открытые исключения получают NEEDS_HUMAN_DECISION. SKU UNKNOWN не меняет
результат сверки дат, количеств и денежных сумм.
"""
dossier = db_dossiers . get_dossier ( dossier_id )
if dossier is None :
return { " ok " : False , " error " : " dossier not found " , " dossier_id " : dossier_id }
window = _window ( target_date_or_period )
items = db_timeline . list_service_items ( dossier_id )
intervals = [
interval for interval in db_timeline . list_timeline ( dossier_id )
if _interval_overlaps ( interval , window )
]
open_exceptions = db_exceptions . list_exceptions ( dossier_id = dossier_id , status = " OPEN " )
exceptions_by_item = { }
for issue in open_exceptions :
service_item_id = issue . get ( " service_item_id " )
if service_item_id :
exceptions_by_item . setdefault ( service_item_id , [ ] ) . append ( issue )
findings = [ ]
observed_items = set ( )
uncertain_items = set ( )
for row_index , external in enumerate ( external_rows or [ ] , start = 1 ) :
if not isinstance ( external , dict ) :
findings . append ( _finding ( " NEEDS_HUMAN_DECISION " , reason = " external row must be an object " , row_index = row_index ) )
continue
match = _find_service ( external , items , catalog_entries or [ ] )
if match [ " status " ] == " AMBIGUOUS " :
candidate_ids = [ item [ " id " ] for item in match [ " candidates " ] ]
uncertain_items . update ( candidate_ids )
findings . append ( _finding (
" NEEDS_HUMAN_DECISION " , external = external , reason = " service match is ambiguous " ,
candidate_service_item_ids = candidate_ids ,
) )
continue
service_item = match . get ( " service_item " )
if service_item is None :
findings . append ( _finding (
" UNCONTRACTED_BILLING " , external = external ,
reason = " no matching service item exists in dossier " ,
) )
continue
observed_items . add ( service_item [ " id " ] )
catalog_status = service_item . get ( " sku_status " ) or " UNKNOWN "
if catalog_entries :
catalog_result = match_catalog_sku (
{ * * service_item , " charge_type " : external . get ( " charge_type " ) } ,
catalog_entries ,
)
catalog_status = catalog_result [ " status " ]
if service_item [ " id " ] in exceptions_by_item :
findings . append ( _finding (
" NEEDS_HUMAN_DECISION " , external = external , service_item = service_item ,
reason = " service item has open dossier exceptions " ,
sku_status = catalog_status ,
exception_ids = [ issue [ " id " ] for issue in exceptions_by_item [ service_item [ " id " ] ] ] ,
) )
continue
matching_intervals = [
interval for interval in intervals
if interval . get ( " service_item_id " ) == service_item [ " id " ]
and ( not external . get ( " charge_type " ) or interval . get ( " charge_type " ) == external . get ( " charge_type " ) )
]
if not matching_intervals :
findings . append ( _finding (
" UNCONTRACTED_BILLING " , external = external , service_item = service_item ,
reason = " service item has no contractual interval in the target window " ,
sku_status = catalog_status ,
) )
continue
if len ( matching_intervals ) > 1 :
exact = [ interval for interval in matching_intervals if _date_comparison ( interval , external ) [ " matches " ] ]
if len ( exact ) == 1 :
matching_intervals = exact
else :
uncertain_items . add ( service_item [ " id " ] )
findings . append ( _finding (
" NEEDS_HUMAN_DECISION " , external = external , service_item = service_item ,
reason = " external period overlaps multiple contractual intervals " ,
sku_status = catalog_status ,
candidate_interval_ids = [ interval [ " id " ] for interval in matching_intervals ] ,
) )
continue
interval = matching_intervals [ 0 ]
dates = _date_comparison ( interval , external )
if not dates [ " matches " ] :
findings . append ( _finding (
" DATE_MISMATCH " , external = external , service_item = service_item ,
interval = interval , reason = " date range differs " , sku_status = catalog_status , * * dates ,
) )
continue
money = _money_comparison ( interval , external )
if not money [ " comparable " ] :
findings . append ( _finding (
" NEEDS_HUMAN_DECISION " , external = external , service_item = service_item ,
interval = interval , reason = money [ " reason " ] , sku_status = catalog_status ,
) )
elif not money [ " matches " ] :
findings . append ( _finding (
" PRICE_OR_QTY_MISMATCH " , external = external , service_item = service_item ,
interval = interval , reason = money [ " reason " ] , sku_status = catalog_status ,
) )
else :
findings . append ( _finding (
" MATCH " , external = external , service_item = service_item , interval = interval ,
reason = " dates, quantity and amount match " , sku_status = catalog_status ,
) )
for interval in intervals :
service_item_id = interval . get ( " service_item_id " )
if service_item_id in observed_items or service_item_id in uncertain_items :
continue
if interval . get ( " charge_type " ) == " payg " :
continue
service_item = next ( ( item for item in items if item . get ( " id " ) == service_item_id ) , None )
if service_item_id in exceptions_by_item :
findings . append ( _finding (
" NEEDS_HUMAN_DECISION " , service_item = service_item , interval = interval ,
reason = " service item has open dossier exceptions " ,
exception_ids = [ issue [ " id " ] for issue in exceptions_by_item [ service_item_id ] ] ,
) )
else :
findings . append ( _finding (
" UNBILLED_SERVICE " , service_item = service_item , interval = interval ,
reason = " contractual service is active but absent from external rows " ,
) )
summary = { }
for finding in findings :
summary [ finding [ " status " ] ] = summary . get ( finding [ " status " ] , 0 ) + 1
return {
" ok " : True ,
" dossier_id " : dossier_id ,
" target " : {
" kind " : window [ " kind " ] ,
" start " : window [ " start " ] . isoformat ( ) ,
" end_exclusive " : window [ " end " ] . isoformat ( ) ,
} ,
" summary " : summary ,
" findings " : findings ,
}
Repinoid