fix: harden extraction batch handling

deploy/nginx-contracts-upload.conf

@@ -5,6 +5,7 @@
 map $http_origin $contracts_upload_cors_origin {
     default "";
     ~^https://contractor\.pythonk8s\.dev\.nubes\.ru$ $http_origin;
+    ~^http://(localhost|127\.0\.0\.1):5001$ $http_origin;
 }
 
 server {

site/config.py

@@ -1,15 +1,51 @@
 """Конфигурация приложения — все настройки в одном месте."""
 import os
 
-VERSION = "2.0.24"
+VERSION = "3.0.3"
+
+
+def _load_env():
+    """Загрузить переменные окружения из .env файла при его наличии.
+
+    Назначение:
+    Позволяет локальному серверу разработки и фоновым скриптам автоматически
+    считывать боевые или тестовые параметры (включая LLM_API_KEY и URL сервисов)
+    без необходимости ручного экспорта в shell.
+
+    Почему так:
+    Порядок поиска обходит текущую директорию site/, родительский каталог
+    contracts-flask/ и корень репозитория. Значения уже установленных системных
+    переменных окружения имеют приоритет и НЕ перезаписываются.
+    """
+    for candidate in [
+        os.path.join(os.path.dirname(__file__), ".env"),
+        os.path.join(os.path.dirname(__file__), "..", ".env"),
+        os.path.join(os.path.dirname(__file__), "..", "..", ".env"),
+    ]:
+        if os.path.isfile(candidate):
+            try:
+                with open(candidate, "r", encoding="utf-8") as f:
+                    for line in f:
+                        line = line.strip()
+                        if line and not line.startswith("#") and "=" in line:
+                            k, v = line.split("=", 1)
+                            k = k.strip()
+                            v = v.strip().strip('"').strip("'")
+                            if k and k not in os.environ:
+                                os.environ[k] = v
+            except Exception:
+                pass
+
+
+_load_env()
 
 LLM_URL = os.getenv("LLM_API_URL", "https://api.aillm.ru/v1/chat/completions")
-LLM_KEY = os.getenv("LLM_API_KEY", "")
+LLM_KEY = os.getenv("LLM_API_KEY") or os.getenv("LLM_KEY", "")
 LLM_MODEL = os.getenv("LLM_MODEL", "gpt-oss-120b")
 
 MAX_CONTENT_LENGTH = 200 * 1024 * 1024  # 200 MB
 API_KEY = os.getenv("API_KEY", "")
-CONVERT_SERVICE_URL = os.getenv("CONVERT_SERVICE_URL", "http://containerk8s.df36c8af-1a95-4623-b551-0d37b731ccca.svc.cluster.local:5000")
+CONVERT_SERVICE_URL = os.getenv("CONVERT_SERVICE_URL", "https://liberta.containerk8s.dev.nubes.ru")
 
 # ── VM-буфер загрузки (паттерн drhider) ──────────────────────────────
 # Браузер кладёт файл на ВМ через WebDAV (мимо шлюза кластера ~64КБ),

site/db/connection.py

@@ -12,7 +12,7 @@ import time
 import os
 from contextlib import contextmanager
 
-DB_PATH = "/tmp/contracts.db"
+DB_PATH = os.getenv("CONTRACTS_DB_PATH", "/tmp/contracts.db")
 _db_session_key = None
 _local = threading.local()
 
@@ -67,6 +67,19 @@ def _create_schema(conn):
             created_at TEXT DEFAULT (datetime('now'))
         );
 
+        CREATE TABLE IF NOT EXISTS extraction_jobs (
+            batch_id TEXT PRIMARY KEY,
+            owner TEXT NOT NULL,
+            state TEXT NOT NULL,
+            total INTEGER NOT NULL,
+            done INTEGER NOT NULL DEFAULT 0,
+            failed INTEGER NOT NULL DEFAULT 0,
+            classes_json TEXT NOT NULL DEFAULT '{}',
+            result_json TEXT,
+            error TEXT,
+            updated_at REAL NOT NULL
+        );
+
         CREATE TABLE IF NOT EXISTS contracts (
             id TEXT PRIMARY KEY,
             number TEXT NOT NULL,
@@ -130,12 +143,137 @@ def _create_schema(conn):
             created_at TEXT DEFAULT (datetime('now'))
         );
 
+        -- Извлечённый паспорт хранится отдельно от исходного документа, чтобы
+        -- повторный запуск пайплайна не перезаписывал исходные байты и парсинг.
+        CREATE TABLE IF NOT EXISTS document_extractions (
+            id TEXT PRIMARY KEY,
+            document_id TEXT NOT NULL UNIQUE REFERENCES documents(id) ON DELETE CASCADE,
+            doc_class TEXT NOT NULL DEFAULT 'unknown',
+            intent_mode TEXT NOT NULL DEFAULT 'no_spec',
+            own_number TEXT,
+            own_date TEXT,
+            counterparty_name TEXT,
+            counterparty_inn TEXT,
+            referenced_numbers_json TEXT NOT NULL DEFAULT '[]',
+            sections_json TEXT NOT NULL DEFAULT '[]',
+            math_validation_json TEXT NOT NULL DEFAULT '{}',
+            status TEXT NOT NULL DEFAULT 'pending',
+            prompt_version TEXT NOT NULL DEFAULT '',
+            raw_llm_response TEXT NOT NULL DEFAULT '{}',
+            created_at TEXT DEFAULT (datetime('now')),
+            updated_at TEXT DEFAULT (datetime('now'))
+        );
+
+        -- Досье создаётся в рамках batch; отсутствие рамочного договора не
+        -- запрещает реконструкцию по спецификациям и дополнительным соглашениям.
+        CREATE TABLE IF NOT EXISTS contract_dossiers (
+            id TEXT PRIMARY KEY,
+            batch_id TEXT,
+            canonical_number TEXT NOT NULL,
+            alias_numbers_json TEXT NOT NULL DEFAULT '[]',
+            counterparty_name TEXT NOT NULL DEFAULT '',
+            counterparty_inn TEXT NOT NULL DEFAULT '',
+            has_frame_contract INTEGER NOT NULL DEFAULT 0,
+            status TEXT NOT NULL DEFAULT 'ACTIVE',
+            created_at TEXT DEFAULT (datetime('now'))
+        );
+
+        -- Связь задаёт принятую принадлежность документа и сохраняет её основание.
+        -- UNIQUE предотвращает дубли при повторной сборке групп одного batch.
+        CREATE TABLE IF NOT EXISTS dossier_documents (
+            id TEXT PRIMARY KEY,
+            dossier_id TEXT NOT NULL REFERENCES contract_dossiers(id) ON DELETE CASCADE,
+            document_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+            role TEXT NOT NULL DEFAULT 'supplement',
+            link_status TEXT NOT NULL DEFAULT 'CONFIRMED',
+            link_reason TEXT NOT NULL DEFAULT '',
+            seq_order INTEGER NOT NULL DEFAULT 0,
+            created_at TEXT DEFAULT (datetime('now')),
+            UNIQUE (dossier_id, document_id)
+        );
+
+        -- Идентичность услуги отделена от параметров и цены, которые меняются
+        -- по временным интервалам.
+        CREATE TABLE IF NOT EXISTS service_items (
+            id TEXT PRIMARY KEY,
+            dossier_id TEXT NOT NULL REFERENCES contract_dossiers(id) ON DELETE CASCADE,
+            service_family TEXT NOT NULL,
+            instance_anchor TEXT NOT NULL DEFAULT '',
+            canonical_name TEXT NOT NULL,
+            catalog_sku TEXT,
+            sku_status TEXT NOT NULL DEFAULT 'UNKNOWN',
+            created_at TEXT DEFAULT (datetime('now'))
+        );
+
+        -- Источник интервала каскадно удаляется вместе с документом. Документ,
+        -- закрывший интервал, очищается через SET NULL: сам интервал остаётся.
+        CREATE TABLE IF NOT EXISTS timeline_intervals (
+            id TEXT PRIMARY KEY,
+            dossier_id TEXT NOT NULL REFERENCES contract_dossiers(id) ON DELETE CASCADE,
+            service_item_id TEXT NOT NULL REFERENCES service_items(id) ON DELETE CASCADE,
+            charge_type TEXT NOT NULL DEFAULT 'recurring',
+            valid_from TEXT,
+            valid_to TEXT,
+            date_status TEXT NOT NULL DEFAULT 'explicit',
+            date_basis TEXT NOT NULL DEFAULT '',
+            raw_name TEXT NOT NULL,
+            config_params_json TEXT NOT NULL DEFAULT '{}',
+            price TEXT,
+            qty TEXT,
+            sum TEXT,
+            vat_rate TEXT,
+            vat_mode TEXT NOT NULL DEFAULT 'with_vat',
+            source_document_id TEXT REFERENCES documents(id) ON DELETE CASCADE,
+            closed_by_document_id TEXT REFERENCES documents(id) ON DELETE SET NULL,
+            table_ref TEXT NOT NULL DEFAULT '',
+            quote TEXT NOT NULL DEFAULT '',
+            status TEXT NOT NULL DEFAULT 'VERIFIED',
+            created_at TEXT DEFAULT (datetime('now'))
+        );
+
+        -- Исключения могут быть привязаны к разным этапам и разрешаться отдельно.
+        CREATE TABLE IF NOT EXISTS exceptions_queue (
+            id TEXT PRIMARY KEY,
+            batch_id TEXT,
+            dossier_id TEXT REFERENCES contract_dossiers(id) ON DELETE CASCADE,
+            document_id TEXT REFERENCES documents(id) ON DELETE CASCADE,
+            service_item_id TEXT REFERENCES service_items(id) ON DELETE SET NULL,
+            severity TEXT NOT NULL DEFAULT 'CONFLICT',
+            stage TEXT NOT NULL,
+            title TEXT NOT NULL,
+            source_quote TEXT NOT NULL DEFAULT '',
+            options_json TEXT NOT NULL DEFAULT '[]',
+            resolution_json TEXT,
+            status TEXT NOT NULL DEFAULT 'OPEN',
+            created_at TEXT DEFAULT (datetime('now')),
+            resolved_at TEXT
+        );
+
         CREATE INDEX IF NOT EXISTS idx_documents_batch ON documents(batch_id);
         CREATE INDEX IF NOT EXISTS idx_documents_hash ON documents(batch_id, content_hash);
         CREATE INDEX IF NOT EXISTS idx_supplements_contract ON supplements(contract_id);
         CREATE INDEX IF NOT EXISTS idx_spec_current_contract ON spec_current(contract_id);
         CREATE INDEX IF NOT EXISTS idx_spec_events_supplement ON spec_events(supplement_id);
+        CREATE INDEX IF NOT EXISTS idx_extractions_document ON document_extractions(document_id);
+        CREATE INDEX IF NOT EXISTS idx_dossiers_batch ON contract_dossiers(batch_id);
+        CREATE INDEX IF NOT EXISTS idx_dossier_documents_dossier ON dossier_documents(dossier_id);
+        CREATE INDEX IF NOT EXISTS idx_dossier_documents_document ON dossier_documents(document_id);
+        CREATE INDEX IF NOT EXISTS idx_service_items_dossier ON service_items(dossier_id);
+        CREATE INDEX IF NOT EXISTS idx_timeline_dossier ON timeline_intervals(dossier_id);
+        CREATE INDEX IF NOT EXISTS idx_timeline_service ON timeline_intervals(service_item_id);
+        CREATE INDEX IF NOT EXISTS idx_exceptions_dossier ON exceptions_queue(dossier_id);
+        CREATE INDEX IF NOT EXISTS idx_exceptions_batch ON exceptions_queue(batch_id);
     """)
+    # Existing /tmp databases survive application restarts. Additive migrations
+    # therefore inspect the live schema before ALTER TABLE, while fresh databases
+    # receive the same column from the CREATE TABLE definition above.
+    timeline_columns = {
+        row["name"] for row in conn.execute("PRAGMA table_info(timeline_intervals)").fetchall()
+    }
+    if "vat_rate" not in timeline_columns:
+        conn.execute("ALTER TABLE timeline_intervals ADD COLUMN vat_rate TEXT")
+        if not getattr(_local, "in_transaction", False):
+            conn.commit()
 
 
 def cleanup_db():
@@ -207,7 +345,10 @@ def execute_returning(sql, params=None):
         sql = sql[:sql.upper().rfind(" RETURNING ")]
 
     cur = conn.execute(sql, params or [])
-    conn.commit()
+    # The outer transaction owns commit and rollback; committing here would make
+    # its rollback ineffective and could expose only part of a multi-write update.
+    if not getattr(_local, "in_transaction", False):
+        conn.commit()
     rowid = cur.lastrowid
 
     if table and rowid:

site/db/documents.py

@@ -109,6 +109,21 @@ def list_by_batch(batch_id):
     )
 
 
+def list_for_extraction(batch_id):
+    """Получить исходный текст парсинга для stateless-обработки batch.
+
+    Отдельный метод не расширяет legacy-ответ `list_by_batch()`, которым уже
+    пользуются старые API и UI. Extractor читает только поля документа и не
+    меняет состояние загрузки или классификации.
+    """
+    return query(
+          """SELECT id, filename, status, elements_json
+              FROM documents WHERE batch_id = %s AND status = 'parsed'
+              ORDER BY created_at, id""",
+        (batch_id,),
+    )
+
+
 def count_by_status(batch_id):
     """Count documents by classify_status."""
     rows = query(

site/db/dossiers.py

@@ -0,0 +1,126 @@
+"""CRUD для досье договоров и доказанных связей с документами.
+
+Досье идентифицируется внутренним UUID, а номер договора и его алиасы
+сохраняются как изменяемые утверждения. Поэтому опечатки в документах
+не становятся ключом базы и могут быть пересмотрены без переноса данных.
+"""
+import json
+import uuid
+
+from db.connection import execute, query
+
+
+def _json_value(value, default):
+    """Кодировать поля-списки, принимая уже сериализованный JSON без повтора."""
+    if value is None:
+        value = default
+    return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
+
+
+def _decode_dossier(row):
+    """Добавить к строке досье Python-представление списка алиасов."""
+    if not row:
+        return None
+    result = dict(row)
+    try:
+        result["alias_numbers"] = json.loads(result.get("alias_numbers_json") or "[]")
+    except (TypeError, json.JSONDecodeError):
+        result["alias_numbers"] = []
+    return result
+
+
+def create_dossier(batch_id, canonical_number, alias_numbers=None,
+                   counterparty_name="", counterparty_inn="",
+                   has_frame_contract=False, status="ACTIVE"):
+    """Создать досье и вернуть его запись со стабильным внутренним ID."""
+    dossier_id = str(uuid.uuid4())
+    execute(
+        """INSERT INTO contract_dossiers
+           (id, batch_id, canonical_number, alias_numbers_json,
+            counterparty_name, counterparty_inn, has_frame_contract, status)
+           VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""",
+        (dossier_id, batch_id, canonical_number,
+         _json_value(alias_numbers, []), counterparty_name or "",
+         counterparty_inn or "", int(bool(has_frame_contract)), status),
+    )
+    return get_dossier(dossier_id)
+
+
+def link_document(dossier_id, document_id, role="supplement",
+                  link_status="CONFIRMED", link_reason="", seq_order=0):
+    """Создать или обновить связь dossier-document идемпотентно.
+
+    Повторная сборка групп не должна создавать дубликаты. При уже имеющейся
+    связи сохраняется её ID, но актуальные роль, статус, основание и порядок
+    заменяются значениями нового решения группировщика.
+    """
+    existing = query(
+        "SELECT id FROM dossier_documents WHERE dossier_id = %s AND document_id = %s",
+        (dossier_id, document_id),
+    )
+    if existing:
+        link_id = existing[0]["id"]
+        execute(
+            """UPDATE dossier_documents
+               SET role = %s, link_status = %s, link_reason = %s, seq_order = %s
+               WHERE id = %s""",
+            (role, link_status, link_reason or "", seq_order, link_id),
+        )
+    else:
+        link_id = str(uuid.uuid4())
+        execute(
+            """INSERT INTO dossier_documents
+               (id, dossier_id, document_id, role, link_status, link_reason, seq_order)
+               VALUES (%s, %s, %s, %s, %s, %s, %s)""",
+            (link_id, dossier_id, document_id, role, link_status,
+             link_reason or "", seq_order),
+        )
+    rows = query("SELECT * FROM dossier_documents WHERE id = %s", (link_id,))
+    return rows[0] if rows else None
+
+
+def list_dossiers_by_batch(batch_id):
+    """Получить досье batch вместе с упорядоченным списком документов."""
+    dossiers = query(
+        "SELECT * FROM contract_dossiers WHERE batch_id = %s ORDER BY created_at, id",
+        (batch_id,),
+    )
+    result = []
+    for raw_dossier in dossiers:
+        dossier = _decode_dossier(raw_dossier)
+        dossier["documents"] = query(
+            """SELECT dd.*, d.filename, d.doc_type, d.own_number, d.parent_number,
+                      d.doc_date, d.counterparty, d.zip_source
+               FROM dossier_documents dd
+               JOIN documents d ON d.id = dd.document_id
+               WHERE dd.dossier_id = %s
+               ORDER BY dd.seq_order, d.created_at, d.id""",
+            (dossier["id"],),
+        )
+        result.append(dossier)
+    return result
+
+
+def get_dossier(dossier_id):
+    """Получить досье по внутреннему ID, не загружая список документов."""
+    rows = query("SELECT * FROM contract_dossiers WHERE id = %s", (dossier_id,))
+    return _decode_dossier(rows[0]) if rows else None
+
+
+def list_dossier_documents(dossier_id):
+    """Вернуть документы досье в установленном порядке вместе с ролью связи."""
+    return query(
+        """SELECT dd.*, d.filename, d.status AS document_status, d.doc_type,
+                  d.own_number, d.parent_number, d.doc_date, d.counterparty,
+                  d.zip_source
+           FROM dossier_documents dd
+           JOIN documents d ON d.id = dd.document_id
+           WHERE dd.dossier_id = %s
+           ORDER BY dd.seq_order, d.created_at, d.id""",
+        (dossier_id,),
+    )
+
+
+def clear_batch_dossiers(batch_id):
+    """Удалить все досье batch; таблицы с ON DELETE CASCADE очистятся вместе с ними."""
+    return execute("DELETE FROM contract_dossiers WHERE batch_id = %s", (batch_id,))

site/db/exceptions.py

@@ -0,0 +1,71 @@
+"""CRUD для очереди исключений и сохранения решений оператора."""
+import json
+import uuid
+
+from db.connection import execute, query
+
+
+def add_exception(batch_id, dossier_id, document_id, service_item_id,
+                  severity, stage, title, source_quote="", options=None):
+    """Добавить конкретную нерешённую задачу и вернуть её запись."""
+    exception_id = str(uuid.uuid4())
+    options_json = json.dumps(options or [], ensure_ascii=False, default=str)
+    execute(
+        """INSERT INTO exceptions_queue
+           (id, batch_id, dossier_id, document_id, service_item_id, severity,
+            stage, title, source_quote, options_json)
+           VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
+        (exception_id, batch_id, dossier_id, document_id, service_item_id,
+         severity, stage, title, source_quote or "", options_json),
+    )
+    return get_exception(exception_id)
+
+
+def get_exception(exception_id):
+    """Получить исключение по ID и декодировать варианты решения."""
+    rows = query("SELECT * FROM exceptions_queue WHERE id = %s", (exception_id,))
+    if not rows:
+        return None
+    result = dict(rows[0])
+    try:
+        result["options"] = json.loads(result.get("options_json") or "[]")
+    except (TypeError, json.JSONDecodeError):
+        result["options"] = []
+    try:
+        result["resolution"] = json.loads(result.get("resolution_json") or "null")
+    except (TypeError, json.JSONDecodeError):
+        result["resolution"] = None
+    return result
+
+
+def list_exceptions(batch_id=None, dossier_id=None, status=None):
+    """Список исключений с необязательными фильтрами batch/dossier/status."""
+    clauses = []
+    params = []
+    for field, value in (("batch_id", batch_id), ("dossier_id", dossier_id),
+                         ("status", status)):
+        if value is not None:
+            clauses.append(f"{field} = %s")
+            params.append(value)
+    where = f" WHERE {' AND '.join(clauses)}" if clauses else ""
+    rows = query(
+        f"SELECT id FROM exceptions_queue{where} ORDER BY created_at, id",
+        params,
+    )
+    return [get_exception(row["id"]) for row in rows]
+
+
+def resolve_exception(exception_id, resolution):
+    """Сохранить решение без удаления исходной причины и цитаты.
+
+    Пересчёт зависимых данных выполняет сервисный слой; CRUD отвечает только
+    за аудируемую фиксацию решения человека.
+    """
+    resolution_json = json.dumps(resolution, ensure_ascii=False, default=str)
+    changed = execute(
+        """UPDATE exceptions_queue
+           SET resolution_json = %s, status = 'RESOLVED', resolved_at = datetime('now')
+           WHERE id = %s AND status = 'OPEN'""",
+        (resolution_json, exception_id),
+    )
+    return get_exception(exception_id) if changed else None

site/db/extraction_jobs.py

@@ -0,0 +1,79 @@
+"""Persistent extraction progress and per-batch worker ownership."""
+import json
+import time
+import uuid
+
+from db.connection import execute, query
+
+LEASE_SECONDS = 300
+
+
+def claim(batch_id, total):
+    """Claim a batch atomically, allowing retry after failure or expired lease."""
+    owner = str(uuid.uuid4())
+    now = time.time()
+    changed = execute(
+        """INSERT INTO extraction_jobs (batch_id, owner, state, total, updated_at)
+           VALUES (%s, %s, 'running', %s, %s)
+           ON CONFLICT(batch_id) DO UPDATE SET owner=excluded.owner, state='running',
+             total=excluded.total, done=0, failed=0, classes_json='{}',
+             result_json=NULL, error=NULL, updated_at=excluded.updated_at
+           WHERE extraction_jobs.state != 'running' OR extraction_jobs.updated_at < %s""",
+        (batch_id, owner, total, now, now - LEASE_SECONDS),
+    )
+    return owner if changed else None
+
+
+def heartbeat(batch_id, owner):
+    return execute(
+        "UPDATE extraction_jobs SET updated_at=%s WHERE batch_id=%s AND owner=%s AND state='running'",
+        (time.time(), batch_id, owner),
+    )
+
+
+def progress(batch_id, owner, completed, result):
+    rows = query(
+        "SELECT failed, classes_json FROM extraction_jobs WHERE batch_id=%s AND owner=%s AND state='running'",
+        (batch_id, owner),
+    )
+    if not rows:
+        return
+    failed = rows[0]["failed"]
+    classes = json.loads(rows[0]["classes_json"])
+    if not result.get("ok"):
+        failed += 1
+    else:
+        doc_class = (result.get("extraction") or {}).get("doc_class", "unknown")
+        classes[doc_class] = classes.get(doc_class, 0) + 1
+    execute(
+        """UPDATE extraction_jobs SET done=%s, failed=%s, classes_json=%s, updated_at=%s
+           WHERE batch_id=%s AND owner=%s AND state='running'""",
+        (completed, failed, json.dumps(classes), time.time(), batch_id, owner),
+    )
+
+
+def finish(batch_id, owner, result=None, error=None):
+    state = "done" if result and result.get("ok") and not error else "failed"
+    return execute(
+        """UPDATE extraction_jobs SET state=%s, result_json=%s, error=%s, updated_at=%s
+           WHERE batch_id=%s AND owner=%s AND state='running'""",
+        (state, json.dumps(result, ensure_ascii=False) if result is not None else None,
+         error or ("extraction failed" if state == "failed" else None), time.time(), batch_id, owner),
+    )
+
+
+def get(batch_id):
+    """Mark an abandoned lease interrupted; callers may explicitly retry it."""
+    execute(
+        """UPDATE extraction_jobs SET state='interrupted', error='worker stopped; retry the batch'
+           WHERE batch_id=%s AND state='running' AND updated_at < %s""",
+        (batch_id, time.time() - LEASE_SECONDS),
+    )
+    rows = query("SELECT * FROM extraction_jobs WHERE batch_id=%s", (batch_id,))
+    if not rows:
+        return None
+    job = rows[0]
+    job["classes"] = json.loads(job.pop("classes_json"))
+    result_json = job.pop("result_json")
+    job["result"] = json.loads(result_json) if result_json else None
+    return job

site/db/extractions.py

@@ -0,0 +1,111 @@
+"""CRUD для результатов stateless-извлечения отдельных документов.
+
+Исходный документ и результат LLM разделены намеренно: повторный запуск
+извлечения может обновить паспорт, не затрагивая байты файла и его структуру.
+Составные поля сохраняются как JSON-текст, поскольку SQLite не требует
+отдельного JSON-типа и существующий слой БД уже использует такой подход.
+"""
+import json
+import uuid
+
+from db.connection import execute, query
+
+
+_JSON_FIELDS = (
+    "referenced_numbers_json",
+    "sections_json",
+    "math_validation_json",
+)
+_FIELDS = (
+    "doc_class",
+    "intent_mode",
+    "own_number",
+    "own_date",
+    "counterparty_name",
+    "counterparty_inn",
+    *_JSON_FIELDS,
+    "status",
+    "prompt_version",
+    "raw_llm_response",
+)
+
+
+def _encode_json(value):
+    """Сериализовать значения в JSON, сохраняя Decimal как точную строку."""
+    return json.dumps(value, ensure_ascii=False, default=str)
+
+
+def _decode_row(row):
+    """Вернуть составные поля паспорта как Python-объекты для сервисов."""
+    if not row:
+        return None
+    result = dict(row)
+    for field in _JSON_FIELDS:
+        raw_value = result.get(field)
+        try:
+            result[field.removesuffix("_json")] = json.loads(raw_value or "null")
+        except (TypeError, json.JSONDecodeError):
+            result[field.removesuffix("_json")] = None
+    return result
+
+
+def upsert_extraction(document_id, data):
+    """Создать или заменить паспорт документа, не меняя его стабильный ID.
+
+    Входные JSON-поля принимают как Python-списки/словари, так и уже
+    сериализованные строки. Поля, которых нет в `data`, получают безопасные
+    значения по умолчанию при вставке и остаются без изменений при обновлении.
+    """
+    existing = query(
+        "SELECT id FROM document_extractions WHERE document_id = %s",
+        (document_id,),
+    )
+    values = {}
+    for field in _FIELDS:
+        if field not in data:
+            continue
+        value = data[field]
+        if field in _JSON_FIELDS and not isinstance(value, str):
+            value = _encode_json(value)
+        values[field] = value
+
+    if existing:
+        if values:
+            assignments = ", ".join(f"{field} = %s" for field in values)
+            params = [*values.values(), document_id]
+            execute(
+                f"UPDATE document_extractions SET {assignments}, "
+                "updated_at = datetime('now') WHERE document_id = %s",
+                params,
+            )
+        return get_by_document(document_id)
+
+    extraction_id = str(uuid.uuid4())
+    columns = ["id", "document_id", *values.keys()]
+    placeholders = ", ".join(["%s"] * len(columns))
+    execute(
+        f"INSERT INTO document_extractions ({', '.join(columns)}) "
+        f"VALUES ({placeholders})",
+        [extraction_id, document_id, *values.values()],
+    )
+    return get_by_document(document_id)
+
+
+def get_by_document(document_id):
+    """Получить паспорт документа или `None`, если он ещё не извлечён."""
+    rows = query(
+        "SELECT * FROM document_extractions WHERE document_id = %s",
+        (document_id,),
+    )
+    return _decode_row(rows[0]) if rows else None
+
+
+def list_by_batch(batch_id):
+    """Получить паспорта всех документов batch в порядке загрузки."""
+    rows = query(
+        """SELECT e.* FROM document_extractions e
+           JOIN documents d ON d.id = e.document_id
+           WHERE d.batch_id = %s ORDER BY d.created_at, d.id""",
+        (batch_id,),
+    )
+    return [_decode_row(row) for row in rows]

site/db/timeline.py

@@ -0,0 +1,143 @@
+"""CRUD для услуг и временных интервалов обязательств.
+
+Денежные поля принимаются как десятичные строки. SQLite здесь отвечает за
+хранение и фильтрацию дат, а арифметика остаётся в сервисном слое на Decimal.
+"""
+import uuid
+
+from db.connection import execute, query, transaction
+
+
+def insert_service_item(dossier_id, service_family, canonical_name,
+                        instance_anchor="", catalog_sku=None,
+                        sku_status="UNKNOWN"):
+    """Создать устойчивую идентичность услуги в пределах досье."""
+    service_id = str(uuid.uuid4())
+    execute(
+        """INSERT INTO service_items
+           (id, dossier_id, service_family, instance_anchor, canonical_name,
+            catalog_sku, sku_status)
+           VALUES (%s, %s, %s, %s, %s, %s, %s)""",
+        (service_id, dossier_id, service_family, instance_anchor or "",
+         canonical_name, catalog_sku, sku_status),
+    )
+    return get_service_item(service_id)
+
+
+def get_service_item(service_item_id):
+    """Получить услугу по ID либо вернуть `None`."""
+    rows = query("SELECT * FROM service_items WHERE id = %s", (service_item_id,))
+    return rows[0] if rows else None
+
+
+def list_service_items(dossier_id):
+    """Получить все идентичности услуг досье в стабильном порядке."""
+    return query(
+        "SELECT * FROM service_items WHERE dossier_id = %s ORDER BY created_at, id",
+        (dossier_id,),
+    )
+
+
+def insert_interval(data):
+    """Создать интервал, сохраняя точные цены/количества строками.
+
+    `data` должен содержать dossier_id, service_item_id и raw_name; остальные
+    поля берутся из значений по умолчанию схемы либо передаются явно.
+    """
+    interval_id = str(uuid.uuid4())
+    fields = (
+        "dossier_id", "service_item_id", "charge_type", "valid_from", "valid_to",
+        "date_status", "date_basis", "raw_name", "config_params_json", "price",
+        "qty", "sum", "vat_mode", "vat_rate", "source_document_id", "closed_by_document_id",
+        "table_ref", "quote", "status",
+    )
+    values = [data[field] for field in fields if field in data]
+    selected_fields = [field for field in fields if field in data]
+    columns = ["id", *selected_fields]
+    placeholders = ", ".join(["%s"] * len(columns))
+    execute(
+        f"INSERT INTO timeline_intervals ({', '.join(columns)}) VALUES ({placeholders})",
+        [interval_id, *values],
+    )
+    rows = query("SELECT * FROM timeline_intervals WHERE id = %s", (interval_id,))
+    return rows[0] if rows else None
+
+
+def close_interval(interval_id, valid_to, closed_by_document_id=None):
+    """Закрыть интервал полуоткрытой границей и указать закрывший документ."""
+    return execute(
+        """UPDATE timeline_intervals
+           SET valid_to = %s, closed_by_document_id = %s
+           WHERE id = %s""",
+        (valid_to, closed_by_document_id, interval_id),
+    )
+
+
+def update_interval(interval_id, values):
+    """Обновить разрешённый набор полей интервала без динамических имён от клиента."""
+    allowed = {
+        "valid_from", "valid_to", "date_status", "date_basis", "raw_name",
+        "config_params_json", "price", "qty", "sum", "vat_mode", "vat_rate", "table_ref",
+        "quote", "status", "closed_by_document_id", "source_document_id",
+    }
+    selected = {field: value for field, value in values.items() if field in allowed}
+    if not selected:
+        return 0
+    assignments = ", ".join(f"{field} = %s" for field in selected)
+    return execute(
+        f"UPDATE timeline_intervals SET {assignments} WHERE id = %s",
+        [*selected.values(), interval_id],
+    )
+
+
+def list_open_intervals(dossier_id, service_item_id=None):
+    """Вернуть открытые интервалы, опционально отфильтрованные по услуге."""
+    if service_item_id is None:
+        return query(
+            """SELECT * FROM timeline_intervals
+               WHERE dossier_id = %s AND valid_to IS NULL
+               ORDER BY valid_from, service_item_id, id""",
+            (dossier_id,),
+        )
+    return query(
+        """SELECT * FROM timeline_intervals
+           WHERE dossier_id = %s AND service_item_id = %s AND valid_to IS NULL
+           ORDER BY valid_from, id""",
+        (dossier_id, service_item_id),
+    )
+
+
+def list_timeline(dossier_id, as_of_date=None):
+    """Получить историю либо активный на дату срез по досье.
+
+    Для среза используется единая полуоткрытая семантика: начало включено,
+    окончание исключено. NULL-дата начала не считается подтверждённо активной.
+    """
+    if as_of_date:
+        return query(
+            """SELECT * FROM timeline_intervals
+               WHERE dossier_id = %s AND valid_from <= %s
+                 AND (valid_to IS NULL OR valid_to > %s)
+               ORDER BY valid_from, service_item_id, id""",
+            (dossier_id, as_of_date, as_of_date),
+        )
+    return query(
+        """SELECT * FROM timeline_intervals WHERE dossier_id = %s
+           ORDER BY valid_from, service_item_id, id""",
+        (dossier_id,),
+    )
+
+
+def reset_dossier_timeline(dossier_id):
+    """Сбросить вычисленный таймлайн и открытые исключения сопоставления.
+
+    Решённые исключения сохраняются как аудит пользовательского решения;
+    нерешённые lineage-исключения пересоздаются вместе с новым построением.
+    """
+    with transaction():
+        execute(
+            """DELETE FROM exceptions_queue
+               WHERE dossier_id = %s AND stage = 'lineage_matching' AND status = 'OPEN'""",
+            (dossier_id,),
+        )
+        return execute("DELETE FROM service_items WHERE dossier_id = %s", (dossier_id,))

site/llm_prompt.py

@@ -279,3 +279,91 @@ def build_classify_prompt(header_text):
 {header_text}
 ---""".replace("{header_text}", header_text)
     return body, ""
+
+
+def build_stateless_extractor_prompt(structured_text):
+    """Собрать изолированный prompt для извлечения фактов из одного документа.
+
+     В отличие от `build_prompt`, эта функция не читает активный prompt из БД,
+     не получает текущую спецификацию и не может подмешать результаты обработки
+     других файлов. Единственный изменяемый вход — текст текущего документа.
+     """
+    instructions = """Ты — извлекатель проверяемых фактов из документов договорного архива ООО «НУБЕС».
+  Проанализируй только один документ ниже. Не используй сведения из других документов,
+  предыдущие версии спецификаций или внешние знания. Верни ровно один JSON-объект без
+  markdown, пояснений, комментариев и текста до или после объекта.
+
+  Схема ответа:
+  {
+    "doc_class": "contract_frame | specification | supplement | ignored_accounting",
+    "intent_mode": "no_spec | full_snapshot | delta_amendment | termination",
+    "own_number": null,
+    "own_date": null,
+    "counterparty": {"name": null, "inn": null},
+    "referenced_contracts": [
+     {"number": "номер как в документе", "date": null, "source": "header | preamble | appendix | other"}
+    ],
+    "sections": [
+     {
+      "table_ref": "table_1",
+      "section_title": null,
+      "charge_type": "one_time | recurring | payg | unknown",
+      "period_start": null,
+      "period_end": null,
+      "header_stated_total": null,
+      "table_footer_total": null,
+      "vat_mode": "with_vat | without_vat | vat_unknown",
+      "vat_rate": null,
+      "rows": [
+        {
+         "row_ref": "table_1_row_1",
+         "raw_name": "наименование дословно",
+         "charge_type": "one_time | recurring | payg | unknown",
+         "unit": null,
+         "qty": null,
+         "price": null,
+         "sum": null,
+         "date_start": null,
+         "date_end": null,
+         "date_status": "explicit | inherited | unknown",
+         "delta_action": "snapshot | add | update | delete | unresolved | unknown",
+         "target_hint": null
+        }
+      ]
+     }
+    ]
+  }
+
+  Правила извлечения:
+  1. Выбирай только значения из этого документа. Для отсутствующих или неразборчивых
+    значений используй JSON null; не угадывай и не восстанавливай данные по контексту.
+  2. Исполнитель всегда ООО «НУБЕС», ИНН 9706005293. Никогда не записывай его в
+    counterparty; counterparty — только другая сторона, если она явно установлена.
+  3. Перечисли все номера и даты договоров из шапки, преамбулы и заголовков приложений
+    в referenced_contracts. Не объединяй разные номера, даже если один выглядит опечаткой.
+    Для contract_frame также продублируй номер самого договора в referenced_contracts.
+    Для specification и supplement обязательно помести родительский номер договора в
+    referenced_contracts; own_number содержит номер самого документа/приложения/ДС,
+    поэтому номер приложения «1» нельзя выдавать за номер договора.
+  4. Для рамочного договора без спецификации укажи intent_mode="no_spec". Для документа,
+    полностью задающего состав услуг, укажи "full_snapshot"; для точечных изменений —
+    "delta_amendment"; для прекращения обязательств — "termination".
+  5. Каждую таблицу услуг верни отдельной секцией. Контекст над таблицей применяй к ней:
+    даты периода наследуй в строки только когда в строке нет собственной даты; тогда
+    date_status="inherited". Иначе ставь "explicit" или "unknown".
+  6. Не включай итоговые строки в rows. Их суммы указывай в table_footer_total;
+    сумму из заголовка секции указывай в header_stated_total.
+    Если ставка НДС указана явно, верни числовой процент в vat_rate; не выводи её из года,
+    контрагента или внешних правил.
+  7. Числа верни JSON-числами с точкой как десятичным разделителем, без валюты и
+    разделителей тысяч. Не вычисляй и не исправляй price, qty, sum или итоги.
+  8. Если документ является счетом, счетом-фактурой, УПД, актом сверки, актом услуг,
+    платежным поручением или иным бухгалтерским документом, укажи
+    doc_class="ignored_accounting", intent_mode="no_spec", пустые sections.
+  9. Не выдумывай target_hint: указывай его только если сам документ однозначно называет
+    изменяемую строку или услугу. Сохраняй исходное написание raw_name.
+
+  СТРУКТУРИРОВАННЫЙ ТЕКСТ ЕДИНСТВЕННОГО ДОКУМЕНТА:
+  ---
+    """
+    return instructions + str(structured_text or "") + "\n---"

site/routes/api_bp.py

@@ -1,14 +1,183 @@
 """API blueprint — groups, documents, supplements, sync, cleanup, spec-current, chat."""
+import csv
+import io
+import json
+from datetime import date
 from flask import Blueprint, request, jsonify
 from db import documents, supplements, spec_current
+from db import dossiers as db_dossiers
+from db import exceptions as db_exceptions
+from db import timeline as db_timeline
+from db import extractions as db_extractions
 from db.connection import execute, query, transaction
 from services.grouping import group_documents, apply_groups
+from services.reconciliation import reconcile_dossier
+from services.timeline_builder import build_dossier_timeline
 from config import LLM_URL, LLM_KEY, LLM_MODEL
 import httpx
 
 api_bp = Blueprint("api", __name__)
 
 
+# ── Contract timeline and exceptions ────────────────────────────
+
+@api_bp.route("/api/dossiers/<dossier_id>/timeline")
+def api_dossier_timeline(dossier_id):
+    """Вернуть полную историю и активный срез обязательств dossier."""
+    dossier = db_dossiers.get_dossier(dossier_id)
+    if not dossier:
+        return jsonify(ok=False, error="dossier not found"), 404
+
+    as_of = request.args.get("as_of") or date.today().isoformat()
+    try:
+        as_of = date.fromisoformat(as_of).isoformat()
+    except ValueError:
+        return jsonify(ok=False, error="as_of must be YYYY-MM-DD"), 400
+
+    history = db_timeline.list_timeline(dossier_id)
+    active = db_timeline.list_timeline(dossier_id, as_of)
+    for interval in history + active:
+        try:
+            interval["config_params"] = json.loads(interval.get("config_params_json") or "{}")
+        except (TypeError, json.JSONDecodeError):
+            interval["config_params"] = {}
+    return jsonify(
+        ok=True,
+        dossier=dossier,
+        as_of=as_of,
+        timeline=history,
+        active=active,
+        exceptions=db_exceptions.list_exceptions(dossier_id=dossier_id, status="OPEN"),
+    )
+
+
+@api_bp.route("/api/dossiers")
+def api_dossiers():
+    """Вернуть dossier cards batch вместе с alias и связанными документами."""
+    batch_id = request.args.get("batch_id")
+    if not batch_id:
+        return jsonify(ok=False, error="batch_id required"), 400
+    dossiers = db_dossiers.list_dossiers_by_batch(batch_id)
+    extractions_by_document = {
+        extraction["document_id"]: extraction
+        for extraction in db_extractions.list_by_batch(batch_id)
+    }
+    for dossier in dossiers:
+        for document in dossier.get("documents", []):
+            extraction = extractions_by_document.get(document.get("document_id"))
+            validation = (extraction or {}).get("math_validation") or {}
+            document["math_status"] = validation.get("status", "UNKNOWN")
+    return jsonify(
+        ok=True,
+        dossiers=dossiers,
+    )
+
+
+@api_bp.route("/api/exceptions")
+def api_exceptions():
+    """Вернуть исключения только по явно заданной области batch или dossier."""
+    batch_id = request.args.get("batch_id")
+    dossier_id = request.args.get("dossier_id")
+    status = request.args.get("status")
+    if not batch_id and not dossier_id:
+        return jsonify(ok=False, error="batch_id or dossier_id required"), 400
+    if status and status not in {"OPEN", "RESOLVED"}:
+        return jsonify(ok=False, error="status must be OPEN or RESOLVED"), 400
+    return jsonify(
+        ok=True,
+        exceptions=db_exceptions.list_exceptions(
+            batch_id=batch_id, dossier_id=dossier_id, status=status,
+        ),
+    )
+
+
+@api_bp.route("/api/reconcile", methods=["POST"])
+def api_reconcile():
+    """Сверить JSON или CSV строки внешнего биллинга с эталоном dossier."""
+    content_type = request.mimetype or ""
+    if content_type == "application/json":
+        body = request.get_json(silent=True) or {}
+        dossier_id = body.get("dossier_id")
+        rows = body.get("external_rows")
+        target = body.get("target_date_or_period")
+        catalog_entries = body.get("catalog_entries", [])
+    elif content_type == "multipart/form-data":
+        dossier_id = request.form.get("dossier_id")
+        upload = request.files.get("file")
+        if upload is None:
+            return jsonify(ok=False, error="CSV file required"), 400
+        try:
+            rows = list(csv.DictReader(io.StringIO(upload.stream.read().decode("utf-8-sig"))))
+        except (UnicodeDecodeError, csv.Error):
+            return jsonify(ok=False, error="CSV must be UTF-8 with a header row"), 400
+        target = request.form.get("target_date_or_period")
+        if target:
+            try:
+                target = json.loads(target)
+            except json.JSONDecodeError:
+                return jsonify(ok=False, error="target_date_or_period must be JSON"), 400
+        catalog_entries = request.form.get("catalog_entries", "[]")
+        try:
+            catalog_entries = json.loads(catalog_entries)
+        except json.JSONDecodeError:
+            return jsonify(ok=False, error="catalog_entries must be JSON"), 400
+    else:
+        return jsonify(ok=False, error="use application/json or multipart/form-data"), 415
+
+    if not isinstance(dossier_id, str) or not dossier_id.strip():
+        return jsonify(ok=False, error="dossier_id required"), 400
+    if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
+        return jsonify(ok=False, error="external_rows must be an array of objects"), 400
+    if not isinstance(catalog_entries, list) or any(not isinstance(row, dict) for row in catalog_entries):
+        return jsonify(ok=False, error="catalog_entries must be an array of objects"), 400
+    try:
+        result = reconcile_dossier(dossier_id, rows, target, catalog_entries)
+    except ValueError as error:
+        return jsonify(ok=False, error=str(error)), 400
+    if not result.get("ok"):
+        return jsonify(result), 404
+    return jsonify(result)
+
+
+@api_bp.route("/api/exceptions/<exception_id>/resolve", methods=["POST"])
+def api_resolve_exception(exception_id):
+    """Сохранить выбор оператора и без LLM пересобрать dossier timeline."""
+    issue = db_exceptions.get_exception(exception_id)
+    if not issue:
+        return jsonify(ok=False, error="exception not found"), 404
+    if issue.get("status") != "OPEN":
+        return jsonify(ok=False, error="exception is not open"), 409
+    body = request.get_json(silent=True) or {}
+    resolution = body.get("resolution")
+    if not isinstance(resolution, dict) or not resolution:
+        return jsonify(ok=False, error="resolution object required"), 400
+
+    options = issue.get("options") or []
+    if "option_index" in resolution:
+        option_index = resolution["option_index"]
+        if type(option_index) is not int or not 0 <= option_index < len(options):
+            return jsonify(ok=False, error="option_index is invalid"), 400
+        resolution = {
+            "option_index": option_index,
+            "selected_option": options[option_index],
+        }
+    elif issue.get("stage") == "lineage_matching" and options:
+        return jsonify(ok=False, error="choose an option_index"), 400
+
+    if issue.get("dossier_id") and not db_dossiers.get_dossier(issue["dossier_id"]):
+        return jsonify(ok=False, error="dossier not found"), 404
+    resolved = db_exceptions.resolve_exception(exception_id, resolution)
+    if not resolved:
+        return jsonify(ok=False, error="exception is not open"), 409
+
+    rebuild = None
+    if issue.get("dossier_id"):
+        rebuild = build_dossier_timeline(issue["dossier_id"])
+        if not rebuild.get("ok"):
+            return jsonify(ok=False, error="timeline rebuild failed", rebuild=rebuild), 500
+    return jsonify(ok=True, exception=resolved, rebuild=rebuild)
+
+
 # ── Supplements ──────────────────────────────────────────────────
 
 @api_bp.route("/api/supplements")

site/routes/pipeline_bp.py

@@ -1,20 +1,77 @@
 """Pipeline blueprint — SSE-сравнение + classify."""
-import json, re, os, threading
+import hashlib, hmac, json, re, os, threading, time
 from flask import Blueprint, request, jsonify, Response, stream_with_context
+from config import API_KEY
 from services.process import run_pipeline
-from services.classify import classify_batch
+from services import extractor as stateless_extractor
+from services.engine import analyze_batch
+from services.grouping import build_dossiers_for_batch
+from services.timeline_builder import build_dossier_timeline
 from llm_prompt import build_prompt
-from db import documents
+from db import documents, extraction_jobs
+from db.connection import transaction
 
 pipeline_bp = Blueprint("pipeline", __name__)
 
-# In-memory lock для classify (замена файлового lock)
-_classify_locks: dict[str, threading.Thread] = {}
+_engine_rate_lock = threading.Lock()
+_engine_rate_hits: dict[str, list[float]] = {}
+_ENGINE_MAX_BODY_BYTES = 10 * 1024 * 1024
+_ENGINE_MAX_DOCUMENTS = 100
+_ENGINE_MAX_ELEMENTS = 20_000
+_ENGINE_MAX_DOCUMENT_BYTES = 2 * 1024 * 1024
+_ENGINE_RATE_WINDOW_SECONDS = 60
+_ENGINE_RATE_LIMIT = 10
 
 
 @pipeline_bp.route("/process-v2", methods=["GET"])
 def process_v2():
     """SSE-стриминг сравнения договоров."""
+    dossier_id = request.args.get("dossier_id")
+    if dossier_id:
+        if not re.fullmatch(
+            r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
+            dossier_id, re.I,
+        ):
+            return jsonify(ok=False, error="invalid dossier_id"), 400
+
+        def generate_timeline():
+            import time as _time
+            started = _time.time()
+            yield ": ok\n\n"
+            yield "data: " + json.dumps({
+                "type": "extract_start", "supplement_id": dossier_id,
+                "filename": "Построение таймлайна",
+            }, ensure_ascii=False) + "\n\n"
+            try:
+                result = build_dossier_timeline(dossier_id)
+                if not result.get("ok"):
+                    yield "data: " + json.dumps({
+                        "type": "error", "message": result.get("error", "timeline build failed"),
+                    }, ensure_ascii=False) + "\n\n"
+                    return
+                yield "data: " + json.dumps({
+                    "type": "timeline", "dossier_id": dossier_id,
+                    "timeline": result.get("timeline", []),
+                    "exceptions": result.get("exceptions", []),
+                    "counts": result.get("counts", {}),
+                }, ensure_ascii=False, default=str) + "\n\n"
+                yield "data: " + json.dumps({
+                    "type": "done", "dossier_id": dossier_id,
+                    "total_time_s": round(_time.time() - started, 2),
+                }, ensure_ascii=False) + "\n\n"
+            except GeneratorExit:
+                return
+            except Exception as error:
+                yield "data: " + json.dumps({
+                    "type": "error", "message": str(error),
+                }, ensure_ascii=False) + "\n\n"
+
+        return Response(
+            stream_with_context(generate_timeline()),
+            content_type="text/event-stream; charset=utf-8",
+            headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
+        )
+
     cid = request.args.get("contract_id")
     if not cid or not re.fullmatch(
         r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', cid, re.I
@@ -53,39 +110,245 @@ def process_v2():
 
 @pipeline_bp.route("/api/classify-batch", methods=["POST"])
 def classify_batch_route():
-    """Запустить классификацию. ≤10 файлов — sync, >10 — async (Thread)."""
-    body = request.get_json()
+    """Выполнить stateless extraction и построить contract dossiers."""
+    body = request.get_json(silent=True) or {}
+    if not isinstance(body, dict):
+        return jsonify(ok=False, error="JSON object required"), 400
     batch_id = body.get("batch_id")
-    if not batch_id:
+    if not isinstance(batch_id, str) or not batch_id.strip():
         return jsonify(ok=False, error="batch_id required"), 400
 
-    # Guard: уже запущена?
-    if batch_id in _classify_locks:
-        t = _classify_locks[batch_id]
-        if t.is_alive():
-            return jsonify(ok=False, error="classify already running"), 409
-        else:
-            del _classify_locks[batch_id]
-
-    pending = documents.list_pending(batch_id)
+    pending = documents.list_for_extraction(batch_id)
     total = len(pending)
     if total == 0:
-        return jsonify(ok=False, error="no pending documents"), 400
+        return jsonify(ok=False, error="no parsed documents"), 400
+    owner = extraction_jobs.claim(batch_id, total)
+    if not owner:
+        return jsonify(ok=False, error="classify already running"), 409
 
-    # Sync для малых батчей
+    # Keep small batches synchronous for the existing browser workflow.
     if total <= 10:
-        result = classify_batch(batch_id)
+        result = _run_extraction_job(batch_id, pending, owner)
         return jsonify(result)
 
-    # Async для больших
+    _start_extraction_job(batch_id, pending, owner)
+    return jsonify(ok=True, total=total, v3=True), 202
+
+
+@pipeline_bp.route("/api/engine/analyze", methods=["POST"])
+def engine_analyze_route():
+    """Accept parsed documents over a versioned, authenticated JSON boundary.
+
+    Validation is completed for the whole payload before opening the write
+    transaction, so malformed input cannot leave a partial batch. The caller
+    supplies parsed elements only; extraction passports and dossier decisions
+    are always produced by the in-process Engine and persisted by its adapters.
+    """
+    if not API_KEY:
+        return jsonify(ok=False, error="engine API is not configured"), 503
+    supplied_key = request.headers.get("X-API-Key", "")
+    if not hmac.compare_digest(supplied_key.encode("utf-8"), API_KEY.encode("utf-8")):
+        return jsonify(ok=False, error="unauthorized"), 401
+    if _engine_rate_limited():
+        return jsonify(ok=False, error="rate limit exceeded"), 429, {"Retry-After": "60"}
+
+    raw_body = request.stream.read(_ENGINE_MAX_BODY_BYTES + 1)
+    if len(raw_body) > _ENGINE_MAX_BODY_BYTES:
+        return jsonify(ok=False, error="request body too large"), 413
+    try:
+        body = json.loads(raw_body) if request.is_json else None
+    except (ValueError, UnicodeDecodeError):
+        body = None
+    if not isinstance(body, dict):
+        return jsonify(ok=False, error="JSON object required"), 400
+    if type(body.get("schema_version")) is not int or body["schema_version"] != 1:
+        return jsonify(ok=False, error="unsupported schema_version"), 400
+
+    batch_id = body.get("batch_id")
+    if not isinstance(batch_id, str) or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", batch_id):
+        return jsonify(ok=False, error="invalid batch_id"), 400
+    source_documents = body.get("documents")
+    if not isinstance(source_documents, list) or not source_documents:
+        return jsonify(ok=False, error="documents must be a non-empty array"), 400
+    if len(source_documents) > _ENGINE_MAX_DOCUMENTS:
+        return jsonify(ok=False, error="too many documents"), 413
+    validated = []
+    for index, source_document in enumerate(source_documents):
+        if not isinstance(source_document, dict):
+            return jsonify(ok=False, error=f"documents[{index}] must be an object"), 400
+        filename = source_document.get("filename")
+        elements = source_document.get("elements")
+        if not isinstance(filename, str) or not filename.strip() or len(filename) > 512:
+            return jsonify(ok=False, error=f"documents[{index}].filename is invalid"), 400
+        if not isinstance(elements, list) or len(elements) > _ENGINE_MAX_ELEMENTS:
+            return jsonify(ok=False, error=f"documents[{index}].elements is invalid"), 400
+        for element_index, element in enumerate(elements):
+            if not _valid_engine_element(element):
+                return jsonify(
+                    ok=False, error=f"documents[{index}].elements[{element_index}] is invalid",
+                ), 400
+        try:
+            elements_json = json.dumps(
+                elements, ensure_ascii=False, sort_keys=True, separators=(",", ":"),
+                allow_nan=False,
+            )
+        except (TypeError, ValueError):
+            return jsonify(ok=False, error=f"documents[{index}].elements is not valid JSON"), 400
+        encoded_elements = elements_json.encode("utf-8")
+        if len(encoded_elements) > _ENGINE_MAX_DOCUMENT_BYTES:
+            return jsonify(ok=False, error=f"documents[{index}] is too large"), 413
+        validated.append((filename.strip(), elements, elements_json, encoded_elements))
+
+    inserted = []
+    with transaction():
+        # Engine API owns a fresh batch per request. Preventing reuse inside
+        # BEGIN IMMEDIATE keeps old parsed rows out of this run and closes the
+        # race between two concurrent requests using the same batch ID.
+        if documents.list_by_batch(batch_id):
+            return jsonify(ok=False, error="batch_id already exists"), 409
+        owner = extraction_jobs.claim(batch_id, len(validated))
+        if not owner:
+            return jsonify(ok=False, error="classify already running"), 409
+        for filename, elements, elements_json, encoded_elements in validated:
+            content_hash = hashlib.sha256(encoded_elements).hexdigest()
+            document = documents.insert(
+                filename, "application/json", None, status="uploaded",
+                batch_id=batch_id, content_hash=content_hash,
+            )
+            documents.set_parsed(document["id"], elements)
+            inserted.append({"id": document["id"], "filename": filename})
+
+    pending = documents.list_for_extraction(batch_id)
+    total = len(pending)
+    if total > 10:
+        _start_extraction_job(batch_id, pending, owner)
+        return jsonify(ok=True, v3=True, total=total, documents=inserted), 202
+
+    result = _run_extraction_job(batch_id, pending, owner)
+    result["documents"] = inserted
+    return jsonify(result)
+
+
+def _valid_engine_element(element):
+    """Validate the parser fields consumed by structured-text formatting."""
+    if not isinstance(element, dict):
+        return False
+    if element.get("type") == "paragraph":
+        return isinstance(element.get("text"), str) and isinstance(element.get("style", ""), str)
+    if element.get("type") != "table":
+        return False
+    rows = element.get("rows")
+    context = element.get("preceding_paragraphs", [])
+    table_index = element.get("table_index")
+    return (
+        isinstance(rows, list)
+        and all(isinstance(row, list) and all(isinstance(cell, str) for cell in row) for row in rows)
+        and isinstance(context, list)
+        and all(isinstance(paragraph, str) for paragraph in context)
+        and (table_index is None or (type(table_index) is int and table_index > 0))
+    )
+
+
+def _start_extraction_job(batch_id, pending, owner):
+    """Run a claimed batch in the background with persistent progress."""
     def _run():
         try:
-            classify_batch(batch_id)
-        finally:
-            _classify_locks.pop(batch_id, None)
+            _run_extraction_job(batch_id, pending, owner)
+        except Exception:
+            pass
 
-    t = threading.Thread(target=_run, daemon=True)
-    _classify_locks[batch_id] = t
-    t.start()
+    thread = threading.Thread(target=_run, daemon=True)
+    try:
+        thread.start()
+    except Exception as error:
+        extraction_jobs.finish(batch_id, owner, error=str(error))
+        raise
 
-    return jsonify(ok=True, total=total), 202
+
+def _engine_rate_limited():
+    """Apply a small per-source fixed-window limit before accepting LLM work."""
+    now = time.monotonic()
+    source = request.remote_addr or "unknown"
+    cutoff = now - _ENGINE_RATE_WINDOW_SECONDS
+    with _engine_rate_lock:
+        for key in list(_engine_rate_hits):
+            recent = [stamp for stamp in _engine_rate_hits[key] if stamp > cutoff]
+            if recent:
+                _engine_rate_hits[key] = recent
+            else:
+                del _engine_rate_hits[key]
+        recent = _engine_rate_hits.get(source, [])
+        if len(recent) >= _ENGINE_RATE_LIMIT:
+            return True
+        recent.append(now)
+        _engine_rate_hits[source] = recent
+    return False
+
+
+def _run_extraction_job(batch_id, pending, owner):
+    """Keep the lease alive during LLM calls and persist every final outcome."""
+    stop = threading.Event()
+
+    def renew_lease():
+        while not stop.wait(15):
+            if not extraction_jobs.heartbeat(batch_id, owner):
+                return
+
+    heartbeat = threading.Thread(target=renew_lease, daemon=True)
+    heartbeat.start()
+    try:
+        result = _run_v3_extraction(
+            batch_id, pending,
+            progress_callback=lambda completed, item: extraction_jobs.progress(batch_id, owner, completed, item),
+        )
+        extraction_jobs.finish(batch_id, owner, result=result)
+        return result
+    except Exception as error:
+        extraction_jobs.finish(batch_id, owner, error=str(error))
+        raise
+    finally:
+        stop.set()
+        heartbeat.join()
+
+
+def _run_v3_extraction(batch_id, pending, progress_callback=None, llm_client=None):
+    """Собрать extraction и dossier summaries в один ответ API."""
+    parsed_documents = [{
+        "id": document["id"],
+        "filename": document["filename"],
+        "elements": stateless_extractor._decode_elements(document.get("elements_json")),
+    } for document in pending]
+    return analyze_batch(
+        batch_id,
+        parsed_documents,
+        extractor=stateless_extractor.extract_batch,
+        dossier_builder=build_dossiers_for_batch,
+        llm_client=llm_client,
+        progress_callback=progress_callback,
+    )
+
+
+@pipeline_bp.route("/api/classify-progress")
+def classify_progress_route():
+    """Вернуть состояние асинхронного stateless extraction batch."""
+    batch_id = request.args.get("batch")
+    if not batch_id:
+        return jsonify(ok=False, error="batch required"), 400
+    job = extraction_jobs.get(batch_id)
+    if not job:
+        return jsonify(ok=False, error="batch extraction not found"), 404
+    response = {
+        "ok": job["state"] not in {"failed", "interrupted"},
+        "v3": True,
+        "finished": job["state"] in {"done", "failed", "interrupted"},
+        "state": job["state"],
+        "done": job["done"],
+        "total": job["total"],
+        "failed": job["failed"],
+        "classes": job["classes"],
+    }
+    if job.get("error"):
+        response["error"] = job["error"]
+    if job.get("result") is not None:
+        response["result"] = job["result"]
+    return jsonify(response)

site/routes/upload_bp.py

@@ -9,22 +9,47 @@ import config
 
 
 def _convert(filename: str, data: bytes) -> bytes:
-    """.doc → .docx через внешний libreoffice-сервис. Возвращает docx-байты."""
-    try:
-        resp = httpx.post(
-            config.CONVERT_SERVICE_URL + "/convert",
-            files={"file": (filename, data, "application/msword")},
-            timeout=120,
-        )
-    except httpx.TimeoutException:
-        raise Exception("conversion timeout")
-    if resp.status_code != 200:
+    """.doc → .docx через внешний libreoffice-сервис с fallback на публичный endpoint.
+
+    Назначение:
+    Преобразование устаревшего бинарного формата Word 97-2003 (.doc) в современный .docx,
+    так как встроенный парсер документов работает исключительно со структурой OpenXML.
+
+    Почему реализован fallback:
+    Внутри Kubernetes-кластера сервис доступен по внутреннему DNS имени
+    (http://containerk8s.df36c8af-1a95-4623-b551-0d37b731ccca.svc.cluster.local:5000),
+    однако при запуске вне кластера (локальный сервер разработки, standalone-тесты)
+    внутреннее доменное имя не резолвится ([Errno -2] Name or service not known).
+    В таком случае запрос автоматически повторяется к публичному ingress-эндпоинту
+    https://liberta.containerk8s.dev.nubes.ru, гарантируя прозрачную работу в любых средах.
+    """
+    urls = [config.CONVERT_SERVICE_URL]
+    fallback = "https://liberta.containerk8s.dev.nubes.ru"
+    if fallback not in urls:
+        urls.append(fallback)
+
+    last_err = None
+    for base_url in urls:
+        if not base_url:
+            continue
         try:
-            err = resp.json().get("error", "conversion failed")
-        except Exception:
-            err = "conversion failed"
-        raise Exception(err)
-    return resp.content
+            resp = httpx.post(
+                base_url.rstrip("/") + "/convert",
+                files={"file": (filename, data, "application/msword")},
+                timeout=120,
+            )
+            if resp.status_code == 200:
+                return resp.content
+            try:
+                last_err = resp.json().get("error", "conversion failed")
+            except Exception:
+                last_err = f"HTTP {resp.status_code}"
+        except httpx.TimeoutException:
+            last_err = "conversion timeout"
+        except Exception as e:
+            last_err = str(e)
+            continue
+    raise Exception(last_err or "conversion failed")
 
 
 def _store_and_parse(filename: str, data: bytes, batch_id, contract_id, zip_source=None, mime_type="application/octet-stream"):

site/services/catalog_matcher.py

@@ -0,0 +1,66 @@
+"""Детерминированное сопоставление услуги с переданным каталогом SKU.
+
+В workspace нет утверждённого каталога коммерческих артикулов, поэтому модуль
+не содержит выдуманных SKU и принимает каталог явным аргументом. Не найденное
+соответствие возвращает `UNKNOWN`, не мешая сверять даты и суммы договора.
+"""
+import re
+
+
+def _normalize_text(value):
+    """Схлопнуть регистр и пунктуацию для точного сравнения названий каталога."""
+    return re.sub(r"[^0-9A-ZА-ЯЁ]", "", str(value or "").upper().replace("Ё", "Е"))
+
+
+def _normalize_sku(value):
+    """Нормализовать SKU, сохраняя значимый PAYG-суффикс `-m`."""
+    text = str(value or "").strip().upper()
+    payg = text.endswith("-M")
+    core = re.sub(r"[^0-9A-ZА-ЯЁ]", "", text[:-2] if payg else text)
+    return core + ("-M" if payg else "")
+
+
+def match_catalog_sku(service_item, catalog_entries):
+    """Найти единственный SKU по явному SKU или точному нормализованному имени.
+
+    PAYG-артикулы с суффиксом `-m` доступны только для тарифа `payg`; обычные
+    абонентские услуги не поглощают такой артикул. При нескольких точных
+    совпадениях возвращается `AMBIGUOUS`, а при пустом каталоге — `UNKNOWN`.
+    """
+    entries = list(catalog_entries or [])
+    charge_type = str(service_item.get("charge_type") or "unknown").lower()
+    service_sku = _normalize_sku(service_item.get("catalog_sku"))
+    service_name = _normalize_text(
+        service_item.get("canonical_name") or service_item.get("raw_name") or "",
+    )
+    matches = []
+    for entry in entries:
+        if not isinstance(entry, dict) or not entry.get("sku"):
+            continue
+        entry_sku = _normalize_sku(entry["sku"])
+        entry_charge = str(entry.get("charge_type") or "unknown").lower()
+        is_payg_sku = entry_sku.endswith("-M")
+        if is_payg_sku and charge_type != "payg":
+            continue
+        if entry_charge not in {"unknown", charge_type}:
+            continue
+        expected_sku = service_sku
+        if charge_type == "payg" and service_sku and not service_sku.endswith("-M"):
+            expected_sku = service_sku + "-M"
+        exact_sku = bool(service_sku and entry_sku in {service_sku, expected_sku})
+        exact_name = bool(
+            service_name
+            and service_name in {
+                _normalize_text(entry.get("name")),
+                *(_normalize_text(alias) for alias in entry.get("aliases", [])),
+            }
+        )
+        if exact_sku or exact_name:
+            matches.append(entry)
+    unique = { _normalize_sku(entry["sku"]): entry for entry in matches }
+    if len(unique) == 1:
+        sku, entry = next(iter(unique.items()))
+        return {"status": "MATCHED", "sku": entry["sku"], "catalog_entry": entry}
+    if unique:
+        return {"status": "AMBIGUOUS", "sku": None, "candidates": list(unique.values())}
+    return {"status": "UNKNOWN", "sku": None, "candidates": []}

site/services/engine.py

@@ -0,0 +1,82 @@
+"""Shared orchestration for the contract reconstruction engine.
+
+This module deliberately accepts the batch's persistence-backed extractor and
+dossier builder as dependencies. Both the browser workflow and the JSON
+integration boundary therefore execute the same business flow in-process,
+while tests can replace either dependency without constructing HTTP calls or
+making network requests.
+"""
+
+
+def analyze_batch(batch_id, documents, extractor, dossier_builder,
+                  progress_callback=None, llm_client=None):
+    """Extract document passports, assemble dossiers, and shape the API result.
+
+    `extractor` owns LLM interaction and persistence of each extraction;
+    `dossier_builder` consumes those saved passports and persists the resulting
+    graph. Keeping the calls in this order is an invariant: grouping must never
+    observe a partially extracted batch. The callback remains optional because
+    small batches run synchronously, while the existing large-batch route uses
+    it to update persistent progress.
+    """
+    extraction = extractor(
+        batch_id,
+        llm_client=llm_client,
+        progress_callback=progress_callback,
+        documents=documents,
+    )
+    # Extractor-level `ok` means every input succeeded; dossier construction only
+    # requires at least one persisted passport and can proceed for that subset.
+    extraction_results = extraction.get("results") or []
+    total = extraction.get("total", len(documents))
+    done = extraction.get("done")
+    if done is None:
+        done = sum(1 for result in extraction_results if result.get("ok"))
+    failed = extraction.get("failed")
+    if failed is None:
+        failed = max(total - done, 0)
+    dossier_result = dossier_builder(batch_id) if done > 0 else {"ok": False}
+    dossier_ok = bool(dossier_result.get("ok"))
+    # Keep result usability separate from completeness so clients can continue
+    # with valid dossiers while still reporting failures explicitly.
+    status = (
+        "failed" if done == 0 or not dossier_ok
+        else "partial" if failed > 0
+        else "completed"
+    )
+    documents_by_id = {document.get("id"): document for document in documents}
+    failed_documents = []
+    for result in extraction_results:
+        if result.get("ok"):
+            continue
+        document_id = result.get("document_id")
+        source_document = documents_by_id.get(document_id, {})
+        failure = {
+            "document_id": document_id,
+            "filename": source_document.get("filename"),
+            "error": result.get("error") or "extraction failed",
+        }
+        if result.get("persistence_error"):
+            failure["persistence_error"] = result["persistence_error"]
+        failed_documents.append(failure)
+    classes = extraction.get("classes", {})
+    math_checks_passed = sum(
+        1 for result in extraction_results
+        if (result.get("extraction") or {}).get("math_validation", {}).get("status") == "VERIFIED"
+    )
+    return {
+        "ok": status != "failed",
+        "status": status,
+        "partial": status == "partial",
+        "v3": True,
+        "done": done,
+        "total": total,
+        "failed": failed,
+        "failed_documents": failed_documents,
+        "classes": classes,
+        "ignored_accounting": classes.get("ignored_accounting", 0),
+        "math_checks_passed": math_checks_passed,
+        "dossier_count": len(dossier_result.get("dossiers", [])),
+        "ambiguous": dossier_result.get("ambiguous", 0),
+        "unlinked": dossier_result.get("unlinked", []),
+    }

site/services/extractor.py

@@ -0,0 +1,353 @@
+"""Stateless-извлечение паспорта и проверка числовых утверждений документа.
+
+Каждый вызов LLM ограничен одним документом. Результат сохраняется отдельно
+от загруженных байтов и legacy classify-полей; ошибка одного документа не
+останавливает остальные документы batch.
+"""
+import json
+import re
+from decimal import Decimal, InvalidOperation
+
+from config import LLM_KEY, LLM_MODEL, LLM_URL
+from db import documents as db_docs
+from db import exceptions as db_exceptions
+from db import extractions as db_extractions
+from llm_prompt import build_stateless_extractor_prompt
+from services.classify import _is_garbage_by_filename, _is_garbage_by_header
+from services.parse import format_elements_with_structure
+
+log = __import__("logging").getLogger(__name__)
+
+PROMPT_VERSION = "stateless-extractor-v1"
+MONEY_TOLERANCE = Decimal("0.05")
+DOC_CLASSES = {"contract_frame", "specification", "supplement", "ignored_accounting"}
+INTENT_MODES = {"no_spec", "full_snapshot", "delta_amendment", "termination"}
+_HOMOGLYPHS = str.maketrans({
+    "A": "А", "B": "В", "C": "С", "E": "Е", "H": "Н", "K": "К",
+    "M": "М", "O": "О", "P": "Р", "T": "Т", "X": "Х", "Y": "У",
+})
+
+_extractor_llm = None
+
+
+def _get_extractor_client():
+    """Создать production-клиент лениво, сохраняя инъекцию FakeLLMClient в тестах."""
+    global _extractor_llm
+    if _extractor_llm is None:
+        from services.llm_client import HttpxLLMClient
+        _extractor_llm = HttpxLLMClient(
+            url=LLM_URL, key=LLM_KEY, model=LLM_MODEL, max_tokens=12000, timeout=120,
+        )
+    return _extractor_llm
+
+
+def _decode_elements(value):
+    """Вернуть parsed elements как список при хранении JSON-текстом или объектом."""
+    if isinstance(value, str):
+        try:
+            value = json.loads(value)
+        except json.JSONDecodeError:
+            return []
+    return value if isinstance(value, list) else []
+
+
+def _decode_llm_json(raw_text):
+    """Декодировать JSON-объект модели, сохраняя десятичные числа как Decimal."""
+    if not isinstance(raw_text, str) or not raw_text.strip():
+        raise ValueError("empty LLM response")
+    text = raw_text.strip()
+    if text.startswith("```"):
+        text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text, flags=re.IGNORECASE).strip()
+    start, end = text.find("{"), text.rfind("}")
+    if start < 0 or end < start:
+        raise ValueError("LLM response does not contain a JSON object")
+    result = json.loads(text[start:end + 1], parse_float=Decimal)
+    if not isinstance(result, dict):
+        raise ValueError("LLM response must be a JSON object")
+    return result
+
+
+def _normalize_number(value):
+    """Схлопнуть регистр, гомоглифы и разделители для проверки коллизий номеров."""
+    if value is None:
+        return ""
+    return re.sub(r"[^0-9А-ЯЁ]", "", str(value).upper().translate(_HOMOGLYPHS))
+
+
+def _as_decimal(value):
+    """Преобразовать JSON-число или строку в конечный Decimal без float-арифметики."""
+    if value is None or isinstance(value, bool):
+        return None
+    try:
+        number = Decimal(str(value))
+    except (InvalidOperation, ValueError, TypeError):
+        return None
+    return number if number.is_finite() else None
+
+
+def _counterparty_without_nubes(counterparty):
+    """Удалить исполнителя из поля контрагента по имени или официальному ИНН."""
+    if not isinstance(counterparty, dict):
+        return {"name": None, "inn": None}
+    name = counterparty.get("name")
+    inn = counterparty.get("inn")
+    if inn is not None:
+        digits = re.sub(r"\D", "", str(inn))
+        if digits == "9706005293":
+            return {"name": None, "inn": None}
+    if name and re.search(r"НУБЕС|NUBES", str(name), re.IGNORECASE):
+        return {"name": None, "inn": None}
+    return {"name": name, "inn": inn}
+
+
+def _ensure_exception(batch_id, document_id, severity, title, source_quote="",
+                      stage="extraction_math"):
+    """Добавить исключение только один раз, включая уже разрешённые записи."""
+    existing = db_exceptions.list_exceptions(batch_id=batch_id)
+    if any(
+        item.get("document_id") == document_id
+        and item.get("stage") == stage
+        and item.get("title") == title
+        for item in existing
+    ):
+        return
+    db_exceptions.add_exception(
+        batch_id=batch_id,
+        dossier_id=None,
+        document_id=document_id,
+        service_item_id=None,
+        severity=severity,
+        stage=stage,
+        title=title,
+        source_quote=source_quote,
+    )
+
+
+def validate_extraction(extracted_dict, doc_id, batch_id):
+    """Очистить контрагента, проверить суммы и вернуть паспорт с отчётом.
+
+    Пропущенные значения не трактуются как нули: сверка итогов выполняется
+    только когда у каждой строки таблицы есть корректно извлечённая сумма.
+    Денежная арифметика остаётся в Decimal от разбора JSON до сравнения.
+    """
+    extracted = dict(extracted_dict)
+    extracted["counterparty"] = _counterparty_without_nubes(extracted.get("counterparty"))
+    validation = {
+        "row_mismatches": [],
+        "footer_mismatches": [],
+        "header_mismatches": [],
+        "internal_number_conflict": False,
+    }
+    references = extracted.get("referenced_contracts") or []
+    numbers = {
+        normalized for reference in references
+        if isinstance(reference, dict)
+        if (normalized := _normalize_number(reference.get("number")))
+    }
+    validation["internal_number_conflict"] = len(numbers) > 1
+
+    for section_index, section in enumerate(extracted.get("sections") or [], start=1):
+        if not isinstance(section, dict):
+            continue
+        rows = section.get("rows") or []
+        row_sums = []
+        all_rows_have_sum = bool(rows)
+        for row_index, row in enumerate(rows, start=1):
+            if not isinstance(row, dict):
+                all_rows_have_sum = False
+                continue
+            price = _as_decimal(row.get("price"))
+            qty = _as_decimal(row.get("qty"))
+            amount = _as_decimal(row.get("sum"))
+            if amount is None:
+                all_rows_have_sum = False
+            else:
+                row_sums.append(amount)
+            if price is None or qty is None or amount is None:
+                continue
+            difference = abs(price * qty - amount)
+            if difference > MONEY_TOLERANCE:
+                row_ref = row.get("row_ref") or f"section_{section_index}_row_{row_index}"
+                mismatch = {
+                    "section_ref": section.get("table_ref") or f"section_{section_index}",
+                    "row_ref": row_ref,
+                    "price": price,
+                    "qty": qty,
+                    "reported_sum": amount,
+                    "difference": difference,
+                }
+                validation["row_mismatches"].append(mismatch)
+                _ensure_exception(
+                    batch_id, doc_id, "CONFLICT", f"MATH_MISMATCH:{row_ref}",
+                    str(row.get("raw_name") or ""),
+                )
+
+        section_total = sum(row_sums, Decimal("0"))
+        table_ref = section.get("table_ref") or f"section_{section_index}"
+        if all_rows_have_sum:
+            for field, mismatch_key, label in (
+                ("table_footer_total", "footer_mismatches", "FOOTER_MISMATCH"),
+                ("header_stated_total", "header_mismatches", "HEADER_MISMATCH"),
+            ):
+                stated_total = _as_decimal(section.get(field))
+                if stated_total is None:
+                    continue
+                difference = abs(section_total - stated_total)
+                if difference > MONEY_TOLERANCE:
+                    mismatch = {
+                        "section_ref": table_ref,
+                        "row_sum_total": section_total,
+                        "stated_total": stated_total,
+                        "difference": difference,
+                    }
+                    validation[mismatch_key].append(mismatch)
+                    _ensure_exception(
+                        batch_id, doc_id, "CONFLICT", f"{label}:{table_ref}",
+                        str(section.get("section_title") or ""),
+                    )
+
+    not_applicable = (
+        extracted.get("intent_mode") == "no_spec"
+        or extracted.get("doc_class") == "ignored_accounting"
+    )
+    has_math_conflict = any(validation[key] for key in (
+        "row_mismatches", "footer_mismatches", "header_mismatches",
+    ))
+    if not_applicable:
+        validation["status"] = "NOT_APPLICABLE"
+        status = "WARNING" if validation["internal_number_conflict"] else "VERIFIED"
+    elif has_math_conflict:
+        validation["status"] = "CONFLICT"
+        status = "CONFLICT"
+    elif validation["internal_number_conflict"]:
+        validation["status"] = "WARNING"
+        status = "WARNING"
+    else:
+        validation["status"] = "VERIFIED"
+        status = "VERIFIED"
+
+    return {"extracted": extracted, "math_validation": validation, "status": status}
+
+
+def _save_extraction(document_id, extracted, validation, status, raw_response="{}"):
+    """Map the public passport shape to normalized extraction-table columns."""
+    counterparty = extracted.get("counterparty") or {}
+    return db_extractions.upsert_extraction(document_id, {
+        "doc_class": extracted.get("doc_class", "unknown"),
+        "intent_mode": extracted.get("intent_mode", "no_spec"),
+        "own_number": extracted.get("own_number"),
+        "own_date": extracted.get("own_date"),
+        "counterparty_name": counterparty.get("name"),
+        "counterparty_inn": counterparty.get("inn"),
+        "referenced_numbers_json": extracted.get("referenced_contracts") or [],
+        "sections_json": extracted.get("sections") or [],
+        "math_validation_json": validation,
+        "status": status,
+        "prompt_version": PROMPT_VERSION,
+        "raw_llm_response": raw_response,
+    })
+
+
+def extract_document(document, llm_client=None, batch_id=None):
+    """Извлечь паспорт одного документа, изолируя ошибки на уровне документа."""
+    document_id = document.get("id") if isinstance(document, dict) else None
+    raw_response = "{}"
+    try:
+        if not document_id:
+            raise ValueError("document id is required")
+        batch_id = batch_id if batch_id is not None else document.get("batch_id")
+        elements = _decode_elements(document.get("elements", document.get("elements_json")))
+        # Formatting is part of this document's failure boundary: corrupt stored
+        # elements must not escape and stop extraction of later batch members.
+        structured_text = format_elements_with_structure(elements)
+        if _is_garbage_by_filename(document.get("filename") or "") or _is_garbage_by_header(structured_text[:2000]):
+            extracted = {
+                "doc_class": "ignored_accounting", "intent_mode": "no_spec",
+                "own_number": None, "own_date": None,
+                "counterparty": {"name": None, "inn": None},
+                "referenced_contracts": [], "sections": [],
+            }
+        elif not elements:
+            extracted = {
+                "doc_class": "unreadable_scan", "intent_mode": "no_spec",
+                "own_number": None, "own_date": None,
+                "counterparty": {"name": None, "inn": None},
+                "referenced_contracts": [], "sections": [],
+            }
+            _ensure_exception(
+                batch_id, document_id, "WARNING", "UNREADABLE_SCAN",
+                document.get("filename") or "", stage="extraction",
+            )
+        else:
+            client = llm_client if llm_client is not None else _get_extractor_client()
+            raw_response = client.complete(build_stateless_extractor_prompt(structured_text))
+            extracted = _decode_llm_json(raw_response)
+
+        if extracted.get("doc_class") not in DOC_CLASSES and extracted.get("doc_class") != "unreadable_scan":
+            raise ValueError(f"unsupported doc_class: {extracted.get('doc_class')!r}")
+        if extracted.get("intent_mode") not in INTENT_MODES:
+            raise ValueError(f"unsupported intent_mode: {extracted.get('intent_mode')!r}")
+        if not isinstance(extracted.get("referenced_contracts", []), list):
+            raise ValueError("referenced_contracts must be an array")
+        if not isinstance(extracted.get("sections", []), list):
+            raise ValueError("sections must be an array")
+
+        result = validate_extraction(extracted, document_id, batch_id)
+        if extracted.get("doc_class") == "unreadable_scan":
+            result["math_validation"]["status"] = "NOT_APPLICABLE"
+            result["status"] = "WARNING"
+        saved = _save_extraction(
+            document_id, result["extracted"], result["math_validation"],
+            result["status"], raw_response,
+        )
+        return {"ok": True, "document_id": document_id, "status": result["status"], "extraction": saved}
+    except Exception as error:
+        log.exception("Stateless extraction failed for document %s", document_id)
+        result = {"ok": False, "document_id": document_id, "error": str(error)}
+        if document_id:
+            # Failure persistence is best-effort; a secondary write failure must
+            # not escape this handler and abort processing of remaining documents.
+            try:
+                _save_extraction(document_id, {
+                    "doc_class": "unknown", "intent_mode": "no_spec", "counterparty": {},
+                    "referenced_contracts": [], "sections": [],
+                }, {"status": "FAILED", "error": str(error)}, "FAILED", raw_response)
+            except Exception as persistence_error:
+                log.exception("Failed to persist extraction failure for document %s", document_id)
+                result["persistence_error"] = str(persistence_error)
+        return result
+
+
+def extract_batch(batch_id, llm_client=None, progress_callback=None, documents=None):
+    """Обработать документы batch и вернуть сводку без batch-wide failure.
+
+    Optional progress callback получает (completed_count, document_result) после
+    каждого документа; ошибки callback не должны останавливать обработку batch.
+    """
+    if documents is None:
+        documents = db_docs.list_for_extraction(batch_id)
+    results = []
+    for completed, document in enumerate(documents, start=1):
+        result = extract_document(document, llm_client=llm_client, batch_id=batch_id)
+        results.append(result)
+        if progress_callback is not None:
+            try:
+                progress_callback(completed, result)
+            except Exception:
+                log.exception("Extraction progress callback failed for batch %s", batch_id)
+    done = sum(1 for result in results if result["ok"])
+    failed = len(results) - done
+    classes = {}
+    for result in results:
+        if result["ok"]:
+            extraction = result.get("extraction") or {}
+            doc_class = extraction.get("doc_class", "unknown")
+            classes[doc_class] = classes.get(doc_class, 0) + 1
+    return {
+        "ok": failed == 0 and bool(results),
+        "total": len(results),
+        "done": done,
+        "failed": failed,
+        "classes": classes,
+        "results": results,
+    }

site/services/grouping.py

@@ -12,6 +12,16 @@ import re
 from db import documents as db_docs
 from db import contracts as db_contracts
 from db import supplements as db_supplements
+from db import dossiers as db_dossiers
+from db import exceptions as db_exceptions
+from db import extractions as db_extractions
+from db.connection import transaction
+
+
+_HOMOGLYPHS = str.maketrans({
+    "A": "А", "B": "В", "C": "С", "E": "Е", "H": "Н", "K": "К",
+    "M": "М", "O": "О", "P": "Р", "T": "Т", "X": "Х", "Y": "У",
+})
 
 
 def normalize_number(num):
@@ -25,6 +35,372 @@ def normalize_number(num):
     return re.sub(r"[^A-Z0-9А-Я]", "", num.upper())
 
 
+def canonicalize_contract_number(number):
+    """Нормализовать полный номер, включая визуальные латинско-кириллические пары."""
+    if not number:
+        return ""
+    normalized = str(number).upper().translate(_HOMOGLYPHS)
+    return re.sub(r"[^0-9А-ЯЁ]", "", normalized)
+
+
+def _is_contract_number(number):
+    """Отличить номер договора от короткого номера приложения или допсоглашения."""
+    key = canonicalize_contract_number(number)
+    return len(key) >= 4 and not (key.isdigit() and int(key) < 1000)
+
+
+def _contract_number_identity(number):
+    """Вернуть полный ключ и ограниченный alias-ключ для формата XXXnnn-number.
+
+    Короткое ядро допускается только при наличии явного клиентского префикса и
+    минимум четырёх цифр, чтобы случайные хвосты вроде «1» не объединяли договоры.
+    Пары двух разных prefixed-номеров по одному ядру не считаются совпадением.
+    """
+    full_key = canonicalize_contract_number(number)
+    normalized = str(number or "").upper().translate(_HOMOGLYPHS)
+    prefix_match = re.match(r"^(?:XXX|ХХХ)\d+[-_/](.+)$", normalized)
+    if not prefix_match:
+        suffix_aliases = [full_key]
+        numbered_suffix = re.match(r"^(.+)_\d+$", normalized)
+        if numbered_suffix:
+            core_key = canonicalize_contract_number(numbered_suffix.group(1))
+            if sum(character.isdigit() for character in core_key) >= 4:
+                suffix_aliases.append(core_key)
+        return {
+            "full": full_key, "suffix": full_key,
+            "suffix_aliases": suffix_aliases, "prefixed": False,
+        }
+    suffix_key = canonicalize_contract_number(prefix_match.group(1))
+    digit_count = sum(character.isdigit() for character in suffix_key)
+    if digit_count < 4:
+        return {
+            "full": full_key, "suffix": full_key,
+            "suffix_aliases": [full_key], "prefixed": True,
+        }
+    return {
+        "full": full_key, "suffix": suffix_key,
+        "suffix_aliases": [suffix_key], "prefixed": True,
+    }
+
+
+def canonicalize_counterparty(name, inn):
+    """Вернуть ключи контрагента для сравнения с жёстким ограничением по ИНН."""
+    inn_key = re.sub(r"\D", "", str(inn or ""))
+    if inn_key and len(inn_key) != 10 and len(inn_key) != 12:
+        inn_key = ""
+    normalized_name = str(name or "").upper().translate(_HOMOGLYPHS)
+    normalized_name = re.sub(r"[«»\"'`.,()\s]", "", normalized_name)
+    normalized_name = re.sub(r"^(ООО|ЗАО|АО|ПАО|ИП)", "", normalized_name)
+    return {"name": normalized_name, "inn": inn_key}
+
+
+def _counterparties_compatible(left, right):
+    """Запретить связь только при конфликте двух явно указанных ИНН."""
+    left_id = canonicalize_counterparty(left.get("name"), left.get("inn"))
+    right_id = canonicalize_counterparty(right.get("name"), right.get("inn"))
+    return not (left_id["inn"] and right_id["inn"] and left_id["inn"] != right_id["inn"])
+
+
+def _passport_keys(passport):
+    """Собрать ключи родительских договоров, отфильтровав номера приложений."""
+    doc_class = passport.get("doc_class")
+    values = []
+    own_number = passport.get("own_number")
+    if doc_class == "contract_frame" and _is_contract_number(own_number):
+        values.append((own_number, "own_number"))
+    elif doc_class == "specification" and _is_contract_number(own_number):
+        values.append((passport["own_number"], "own_number"))
+    for reference in passport.get("referenced_numbers") or []:
+        if isinstance(reference, dict) and reference.get("number"):
+            values.append((reference["number"], reference.get("source") or "referenced_contract"))
+    unique = {}
+    for raw, source in values:
+        identity = _contract_number_identity(raw)
+        key = identity["full"]
+        if key and _is_contract_number(key):
+            unique.setdefault(key, {"raw": str(raw), "source": str(source), **identity})
+    return unique
+
+
+def _shared_contract_match(left_keys, right_keys):
+    """Найти безопасное совпадение полного ключа или его допустимого ядра."""
+    for left_key, left in left_keys.items():
+        for right_key, right in right_keys.items():
+            if left_key == right_key:
+                return left_key, left, right
+            if left.get("prefixed") and not right.get("prefixed"):
+                same_suffix = left.get("suffix") in right.get("suffix_aliases", [])
+            elif right.get("prefixed") and not left.get("prefixed"):
+                same_suffix = right.get("suffix") in left.get("suffix_aliases", [])
+            else:
+                same_suffix = left.get("suffix") == right.get("suffix")
+            if same_suffix and (left.get("prefixed") or right.get("prefixed")):
+                if left.get("prefixed") and right.get("prefixed"):
+                    continue
+                return f"alias:{left['suffix']}", left, right
+    return None
+
+
+def _number_sort_key(number):
+    """Разбить номер на текстовые и числовые части для сортировки ДС по номеру."""
+    return tuple((0, int(part)) if part.isdigit() else (1, part.casefold())
+                 for part in re.split(r"(\d+)", str(number or "")))
+
+
+def _dossier_exception(batch_id, document_id, title, source_quote):
+    """Создать устойчивую запись исключения, не дублируя её при пересборке batch."""
+    existing = db_exceptions.list_exceptions(batch_id=batch_id)
+    if any(
+        item.get("document_id") == document_id
+        and item.get("stage") == "dossier_linking"
+        and item.get("title") == title
+        for item in existing
+    ):
+        return
+    db_exceptions.add_exception(
+        batch_id=batch_id, dossier_id=None, document_id=document_id,
+        service_item_id=None, severity="AMBIGUOUS", stage="dossier_linking",
+        title=title, source_quote=source_quote,
+    )
+
+
+def build_dossiers_for_batch(batch_id):
+    """Atomically rebuild the dossier graph, retaining the old graph on failure."""
+    with transaction():
+        return _build_dossiers_for_batch(batch_id)
+
+
+def _build_dossiers_for_batch(batch_id):
+    """Построить и сохранить dossier graph из паспортов документов одного batch.
+
+    Полные номера и ссылки образуют рёбра только при совместимом контрагенте.
+    Компоненты дополнительно защищены от транзитивного слияния двух разных ИНН;
+    документ, который остаётся кандидатом сразу для нескольких таких компонент,
+    исключается из автоматических связей и передаётся в очередь исключений.
+    """
+    documents = db_docs.list_by_batch(batch_id)
+    passports = db_extractions.list_by_batch(batch_id)
+    passport_by_id = {row["document_id"]: row for row in passports}
+    records = []
+    for document in documents:
+        passport = passport_by_id.get(document["id"])
+        # A missing or failed passport says extraction produced no usable evidence;
+        # do not misreport that technical failure as a missing contract number.
+        if not passport or passport.get("status") == "FAILED":
+            continue
+        if passport.get("doc_class") == "ignored_accounting":
+            continue
+        counterparty = {
+            "name": passport.get("counterparty_name"),
+            "inn": passport.get("counterparty_inn"),
+        }
+        records.append({
+            "document": document,
+            "passport": passport,
+            "counterparty": counterparty,
+            "keys": _passport_keys(passport),
+        })
+
+    parent = list(range(len(records)))
+    component_inns = []
+    for record in records:
+        counterparty_id = canonicalize_counterparty(
+            record["counterparty"].get("name"), record["counterparty"].get("inn"),
+        )
+        component_inns.append({counterparty_id["inn"]} if counterparty_id["inn"] else set())
+
+    def find(index):
+        while parent[index] != index:
+            parent[index] = parent[parent[index]]
+            index = parent[index]
+        return index
+
+    candidates = [[] for _ in records]
+    for left_index, left in enumerate(records):
+        for right_index in range(left_index + 1, len(records)):
+            right = records[right_index]
+            match = _shared_contract_match(left["keys"], right["keys"])
+            if not match or not _counterparties_compatible(left["counterparty"], right["counterparty"]):
+                continue
+            shared_key, left_number, right_number = match
+            candidates[left_index].append((right_index, shared_key, left_number["raw"]))
+            candidates[right_index].append((left_index, shared_key, right_number["raw"]))
+            left_root, right_root = find(left_index), find(right_index)
+            if left_root == right_root:
+                continue
+            merged_inns = component_inns[left_root] | component_inns[right_root]
+            if len(merged_inns) > 1:
+                continue
+            parent[right_root] = left_root
+            component_inns[left_root] = merged_inns
+
+    ambiguous_indices = set()
+    for index, edges in enumerate(candidates):
+        key_targets = {}
+        for other, shared_key, _raw_number in edges:
+            key_targets.setdefault(shared_key, set()).add(find(other))
+        candidate_roots = set().union(*key_targets.values()) if key_targets else set()
+        if len(candidate_roots) > 1:
+            ambiguous_indices.add(index)
+            matching_numbers = sorted({
+                raw_number for _other, _key, raw_number in edges
+            })
+            _dossier_exception(
+                batch_id, records[index]["document"]["id"],
+                "AMBIGUOUS_DOSSIER_LINK", ", ".join(matching_numbers),
+            )
+
+    component_members = {}
+    for index, record in enumerate(records):
+        if index in ambiguous_indices:
+            continue
+        if not record["keys"]:
+            _dossier_exception(
+                batch_id, record["document"]["id"], "NO_RELIABLE_CONTRACT_NUMBER",
+                record["document"].get("filename") or "",
+            )
+            continue
+        component_members.setdefault(find(index), []).append(record)
+
+    db_dossiers.clear_batch_dossiers(batch_id)
+    dossier_results = []
+    for members in component_members.values():
+        frames = [r for r in members if r["passport"].get("doc_class") == "contract_frame"]
+        specifications = [r for r in members if r["passport"].get("doc_class") == "specification"]
+        specifications.sort(key=lambda r: (
+            r["passport"].get("own_date") or r["document"].get("doc_date") or "9999",
+            r["document"].get("id", ""),
+        ))
+        identifiers = []
+        for record in members:
+            for key, value in record["keys"].items():
+                identifiers.append((key, value["raw"], value["source"]))
+        canonical_candidates = [
+            r["passport"].get("own_number") for r in frames
+            if _is_contract_number(r["passport"].get("own_number"))
+        ]
+        canonical_candidates.extend(
+            reference.get("number")
+            for record in [*frames, *specifications]
+            for reference in (record["passport"].get("referenced_numbers") or [])
+            if isinstance(reference, dict) and _is_contract_number(reference.get("number"))
+        )
+        canonical_candidates.extend(
+            r["passport"].get("own_number") for r in specifications
+            if _is_contract_number(r["passport"].get("own_number"))
+        )
+        canonical_candidates.extend(
+            raw for _key, raw, _source in identifiers if _is_contract_number(raw)
+        )
+        canonical_number = canonical_candidates[0] if canonical_candidates else ""
+        canonical_key = canonicalize_contract_number(canonical_number)
+        aliases = list(dict.fromkeys(
+            raw for key, raw, _source in identifiers
+            if key != canonical_key and raw != canonical_number
+        ))
+        party_candidates = [
+            record for record in members
+            if record["counterparty"].get("inn") or record["counterparty"].get("name")
+        ]
+        party_rank = {"contract_frame": 0, "supplement": 1, "specification": 2}
+        party_candidates.sort(key=lambda record: party_rank.get(
+            record["passport"].get("doc_class"), 3,
+        ))
+        counterparty_record = party_candidates[0]["counterparty"] if party_candidates else {}
+        dossier = db_dossiers.create_dossier(
+            batch_id=batch_id,
+            canonical_number=canonical_number,
+            alias_numbers=aliases,
+            counterparty_name=counterparty_record.get("name") or "",
+            counterparty_inn=counterparty_record.get("inn") or "",
+            has_frame_contract=bool(frames),
+        )
+        base_spec_id = specifications[0]["document"]["id"] if specifications else None
+        members.sort(key=lambda r: _dossier_order_key(r, base_spec_id))
+        linked_documents = []
+        for sequence, record in enumerate(members, start=1):
+            document = record["document"]
+            passport = record["passport"]
+            doc_class = passport.get("doc_class")
+            if doc_class == "contract_frame":
+                role = "frame_contract"
+            elif doc_class == "specification" and document["id"] == base_spec_id:
+                role = "base_spec"
+            elif doc_class == "specification":
+                role = "specification"
+            elif doc_class == "supplement":
+                role = "supplement"
+            else:
+                role = "document"
+            reason = _link_reason(record, canonical_key, canonical_number)
+            linked_party = canonicalize_counterparty(
+                record["counterparty"].get("name"), record["counterparty"].get("inn"),
+            )
+            selected_party = canonicalize_counterparty(
+                counterparty_record.get("name"), counterparty_record.get("inn"),
+            )
+            if linked_party["name"] and selected_party["name"] and linked_party["name"] != selected_party["name"]:
+                reason += (
+                    f"; расхождение контрагента: {record['counterparty']['name']}"
+                    f" (в досье: {counterparty_record['name']})"
+                )
+            link = db_dossiers.link_document(
+                dossier_id=dossier["id"], document_id=document["id"], role=role,
+                link_status="CONFIRMED", link_reason=reason, seq_order=sequence,
+            )
+            linked_documents.append({**document, "role": role, "link_reason": reason, "link": link})
+        dossier_results.append({**dossier, "documents": linked_documents})
+
+    return {
+        "ok": True,
+        "batch_id": batch_id,
+        "total_docs": len(documents),
+        "dossiers": dossier_results,
+        "unlinked": sorted(records[i]["document"]["id"] for i in ambiguous_indices)
+        + [r["document"]["id"] for r in records if not r["keys"]],
+        "ambiguous": len(ambiguous_indices),
+    }
+
+
+def _dossier_order_key(record, base_spec_id):
+    """Упорядочить документы так, чтобы рамка и базовая спека шли первыми."""
+    document = record["document"]
+    passport = record["passport"]
+    doc_class = passport.get("doc_class")
+    if doc_class == "contract_frame":
+        rank = 0
+    elif doc_class == "specification" and document["id"] == base_spec_id:
+        rank = 1
+    elif doc_class == "specification":
+        rank = 2
+    elif doc_class == "supplement":
+        rank = 3
+    else:
+        rank = 4
+    if doc_class == "supplement":
+        secondary = (_number_sort_key(passport.get("own_number")),
+                     passport.get("own_date") or document.get("doc_date") or "9999")
+    else:
+        secondary = (passport.get("own_date") or document.get("doc_date") or "9999",)
+    return rank, secondary, document.get("id", "")
+
+
+def _link_reason(record, canonical_key, canonical_number):
+    """Сформировать аудируемое основание связи из совпавшей ссылки документа."""
+    for reference in record["passport"].get("referenced_numbers") or []:
+        if not isinstance(reference, dict) or not reference.get("number"):
+            continue
+        if canonicalize_contract_number(reference["number"]) == canonical_key:
+            source = reference.get("source")
+            if source in {"header", "preamble", "appendix"}:
+                return f"Связано по номеру {reference['number']} из {source}"
+            return f"Связано по номеру {reference['number']}"
+    own_number = record["passport"].get("own_number")
+    if own_number and canonicalize_contract_number(own_number) == canonical_key:
+        return f"Собственный номер документа совпадает с {canonical_number}"
+    return f"Связано в dossier по номеру {canonical_number}"
+
+
 def group_documents(batch_id):
     """
     Сгруппировать классифицированные документы по контрактам.

site/services/llm_client.py

@@ -29,7 +29,19 @@ class HttpxLLMClient:
         self.timeout = timeout
 
     def complete(self, prompt: str) -> str:
+        """Отправить запрос к модели LLM через HTTP POST и вернуть текстовое содержимое ответа.
+
+        Назначение и инварианты:
+        - Проверяет обязательное наличие непустого API-ключа перед формированием HTTP-запроса.
+        - Если ключ отсутствует или пуст, выбрасывает явное исключение ValueError вместо передачи
+          невалидного заголовка Authorization (например 'Bearer '), приводящего к ошибке
+          'Illegal header value' на уровне транспортной библиотеки httpx/httpcore.
+        - Обрезает случайные пробельные символы и переносы строк по краям токена.
+        """
         import httpx
+        clean_key = (self.key or "").strip()
+        if not clean_key:
+            raise ValueError("LLM_API_KEY is not configured or empty")
         payload = {
             "model": self.model,
             "messages": [{"role": "user", "content": prompt}],
@@ -41,7 +53,7 @@ class HttpxLLMClient:
                 self.url,
                 json=payload,
                 headers={
-                    "Authorization": f"Bearer {self.key}",
+                    "Authorization": f"Bearer {clean_key}",
                     "Content-Type": "application/json",
                 },
             )

site/services/parse.py

@@ -1,4 +1,9 @@
-"""Parse service — PDF (pdfplumber), DOCX (python-docx), plain text fallback."""
+"""Структурный парсер PDF/DOCX с порядком чтения и контекстом таблиц.
+
+Формат elements_json остаётся совместимым с текущей загрузкой: верхнеуровневый
+status по-прежнему только `parsed` или `error`. Новые метаданные добавляются
+в элементы, чтобы выбор файлов и отображение результата загрузки не менялись.
+"""
 import io
 
 
@@ -19,37 +24,126 @@ def parse_file(filename, data):
 
 
 def _parse_pdf(data):
-    """Parse PDF with pdfplumber — preserves table structure (unlike PyPDF2)."""
+    """Извлечь PDF в порядке страницы, привязав предшествующий текст к таблицам.
+
+    Координаты pdfplumber позволяют отделить заголовки таблиц от текста в её
+    ячейках и расположить абзацы/таблицы в порядке чтения. Для сканов без
+    текстового слоя возвращается parsed с отдельным флагом, чтобы существующий
+    upload-контракт не трактовал нечитаемый файл как ошибку передачи.
+    """
     import pdfplumber
     elements = []
+    table_index = 0
     with pdfplumber.open(io.BytesIO(data)) as pdf:
         for page in pdf.pages:
-            # Tables first — preserves column structure critical for specs
-            tables = page.extract_tables()
-            for table in tables:
-                if table:
-                    rows = []
-                    for row in table:
-                        if row:
-                            cells = [str(cell or "").strip() for cell in row]
-                            if any(cells):
-                                rows.append(cells)
-                    if rows:
-                        elements.append({"type": "table", "rows": rows})
-            # Remaining text as paragraphs
-            text = page.extract_text()
-            if text:
-                for line in text.split("\n"):
-                    line = line.strip()
-                    if line:
-                        elements.append({"type": "paragraph", "text": line, "style": ""})
-    return {"status": "parsed", "element_count": len(elements), "elements": elements}
+            table_events = []
+            table_regions = []
+            for table in page.find_tables():
+                rows = _clean_table_rows(table.extract())
+                if not rows:
+                    continue
+                table_index += 1
+                x0, top, x1, bottom = table.bbox
+                table_regions.append((x0, top, x1, bottom))
+                table_events.append((float(top), {
+                    "type": "table",
+                    "table_index": table_index,
+                    "rows": rows,
+                }))
+
+            word_events = _pdf_paragraph_events(page, table_regions)
+            events = [(top, 0, element) for top, element in word_events]
+            events.extend((top, 1, element) for top, element in table_events)
+            events.sort(key=lambda event: (event[0], event[1]))
+
+            preceding_paragraphs = []
+            for _top, _kind, element in events:
+                if element["type"] == "paragraph":
+                    elements.append(element)
+                    preceding_paragraphs.append(element["text"])
+                    preceding_paragraphs = preceding_paragraphs[-4:]
+                else:
+                    element["preceding_paragraphs"] = list(preceding_paragraphs)
+                    elements.append(element)
+                    preceding_paragraphs = []
+
+    result = {"status": "parsed", "element_count": len(elements), "elements": elements}
+    if not elements:
+        result["is_unreadable_scan"] = True
+    return result
+
+
+def _clean_table_rows(table):
+    """Нормализовать ячейки таблицы, удаляя только полностью пустые строки."""
+    rows = []
+    for row in table or []:
+        if not row:
+            continue
+        cells = [str(cell or "").strip() for cell in row]
+        if any(cells):
+            rows.append(cells)
+    return rows
+
+
+def _pdf_paragraph_events(page, table_regions):
+    """Сгруппировать слова PDF в строки вне bbox таблиц.
+
+    Каждая строка получает верхнюю координату для общей сортировки с таблицами.
+    Если PDF-библиотека смогла получить текст страницы, но не вернула слова,
+    весь текст сохраняется одним абзацем вместо ошибочной пометки «скан».
+    """
+    words = page.extract_words() or []
+    visible_words = []
+    for word in words:
+        x0 = float(word.get("x0", 0))
+        x1 = float(word.get("x1", x0))
+        top = float(word.get("top", 0))
+        bottom = float(word.get("bottom", top))
+        center_x = (x0 + x1) / 2
+        center_y = (top + bottom) / 2
+        inside_table = any(
+            left <= center_x <= right and upper <= center_y <= lower
+            for left, upper, right, lower in table_regions
+        )
+        text = str(word.get("text", "")).strip()
+        if text and not inside_table:
+            visible_words.append((top, x0, text))
+
+    visible_words.sort(key=lambda item: (item[0], item[1]))
+    lines = []
+    for top, x0, text in visible_words:
+        if not lines or abs(top - lines[-1]["top"]) > 3:
+            lines.append({"top": top, "words": []})
+        lines[-1]["words"].append((x0, text))
+
+    events = []
+    for line in lines:
+        line_text = " ".join(text for _x0, text in sorted(line["words"]))
+        if line_text:
+            events.append((line["top"], {
+                "type": "paragraph", "text": line_text, "style": "",
+            }))
+    if events:
+        return events
+    if words and table_regions:
+        return []
+
+    text = page.extract_text()
+    if not text:
+        return []
+    return [
+        (0.0, {"type": "paragraph", "text": line.strip(), "style": ""})
+        for line in text.split("\n") if line.strip()
+    ]
 
 
 def _parse_docx(data):
+    """Извлечь DOCX, сохранив порядок блоков и абзацы перед каждой таблицей."""
     import docx
     doc = docx.Document(io.BytesIO(data))
     elements = []
+    preceding_paragraphs = []
+    table_index = 0
 
     for block in doc.element.body:
         tag = block.tag.split("}")[-1] if "}" in block.tag else block.tag
@@ -57,6 +151,8 @@ def _parse_docx(data):
             text = _extract_paragraph_text(block)
             if text:
                 elements.append({"type": "paragraph", "text": text, "style": ""})
+                preceding_paragraphs.append(text)
+                preceding_paragraphs = preceding_paragraphs[-4:]
         elif tag == "tbl":
             rows = []
             for tr in block.findall(".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}tr"):
@@ -67,7 +163,14 @@ def _parse_docx(data):
                 if cells:
                     rows.append(cells)
             if rows:
-                elements.append({"type": "table", "rows": rows})
+                table_index += 1
+                elements.append({
+                    "type": "table",
+                    "table_index": table_index,
+                    "preceding_paragraphs": list(preceding_paragraphs),
+                    "rows": rows,
+                })
+                preceding_paragraphs = []
 
     return {"status": "parsed", "element_count": len(elements), "elements": elements}
 
@@ -85,3 +188,36 @@ def _parse_text(data):
     lines = [l.strip() for l in text.split("\n") if l.strip()]
     return {"status": "parsed", "element_count": len(lines),
             "elements": [{"type": "paragraph", "text": l, "style": ""} for l in lines]}
+
+
+def format_elements_with_structure(elements):
+    """Отформатировать элементы документа для stateless-промпта.
+
+    Табличный контекст продублирован в заголовке таблицы намеренно: extractor
+    получает локальную связь периода/итога с таблицей, даже если его окно
+    внимания не удерживает длинный юридический текст перед ней.
+    """
+    lines = []
+    paragraph_index = 0
+    table_index = 0
+    for element in elements or []:
+        if not isinstance(element, dict):
+            continue
+        if element.get("type") == "paragraph":
+            text = str(element.get("text") or "").strip()
+            if text:
+                paragraph_index += 1
+                lines.append(f"[ПАРАГРАФ p{paragraph_index}] {text}")
+        elif element.get("type") == "table":
+            table_index += 1
+            table_ref = element.get("table_index") or table_index
+            context = " | ".join(
+                str(value).strip()
+                for value in element.get("preceding_paragraphs", [])
+                if str(value).strip()
+            )
+            lines.append(f"[ТАБЛИЦА #{table_ref} | Контекст над таблицей: {context!r}]")
+            for row_index, row in enumerate(element.get("rows", []), start=1):
+                cells = [str(cell or "").strip() for cell in (row or [])]
+                lines.append(f"  [Строка {table_ref}.{row_index}] " + " | ".join(cells))
+    return "\n".join(lines)

site/services/reconciliation.py

@@ -0,0 +1,381 @@
+"""Сверка внешней CRM/1С-выгрузки с утверждёнными интервалами dossier.
+
+Сверка выполняется по исходным суммам и датам на Decimal/date. Отсутствие
+известного SKU не блокирует проверку договора; неизвестный VAT rate, напротив,
+не позволяет объявить цены совпавшими, если стороны указали разные VAT modes.
+"""
+import re
+from datetime import date, datetime, timedelta
+from decimal import Decimal, InvalidOperation, ROUND_HALF_UP
+
+from db import dossiers as db_dossiers
+from db import exceptions as db_exceptions
+from db import timeline as db_timeline
+from services.catalog_matcher import match_catalog_sku
+from services.metrics import normalize_date
+from services.service_identity import analyze_service_row, match_service_item
+
+
+MONEY_TOLERANCE = Decimal("0.01")
+
+
+def _decimal(value):
+    """Преобразовать внешнее или договорное число в конечный Decimal."""
+    if value is None or value == "" or isinstance(value, bool):
+        return None
+    try:
+        result = Decimal(str(value).replace("\xa0", "").replace(" ", "").replace(",", "."))
+    except (InvalidOperation, ValueError, TypeError):
+        return None
+    return result if result.is_finite() else None
+
+
+def _date(value):
+    """Разобрать поддерживаемую ISO/DD.MM.YYYY дату без принятия мусорных значений."""
+    if not value:
+        return None
+    try:
+        return datetime.strptime(normalize_date(str(value).strip()), "%Y-%m-%d").date()
+    except (TypeError, ValueError):
+        return None
+
+
+def _window(target_date_or_period):
+    """Нормализовать точечную дату или полуоткрытый период [start, end)."""
+    if target_date_or_period is None:
+        point = date.today()
+        return {"kind": "point", "date": point, "start": point, "end": point + timedelta(days=1)}
+    if isinstance(target_date_or_period, (tuple, list)) and len(target_date_or_period) == 2:
+        start, end = map(_date, target_date_or_period)
+    elif isinstance(target_date_or_period, dict):
+        start = _date(target_date_or_period.get("start") or target_date_or_period.get("period_start"))
+        end = _date(target_date_or_period.get("end") or target_date_or_period.get("period_end"))
+    else:
+        point = _date(target_date_or_period)
+        if point is None:
+            raise ValueError("target date must be YYYY-MM-DD or DD.MM.YYYY")
+        return {"kind": "point", "date": point, "start": point, "end": point + timedelta(days=1)}
+    if start is None or end is None or end <= start:
+        raise ValueError("target period must have valid start < end")
+    return {"kind": "period", "date": None, "start": start, "end": end}
+
+
+def _interval_overlaps(interval, window):
+    """Проверить активность интервала на дату или пересечение с period window."""
+    start = _date(interval.get("valid_from"))
+    end = _date(interval.get("valid_to"))
+    if start is None:
+        return False
+    return start < window["end"] and (end is None or end > window["start"])
+
+
+def _external_end(row):
+    """Согласовать inclusive date_end CRM со stored exclusive valid_to."""
+    exclusive = _date(row.get("date_end_exclusive"))
+    if exclusive:
+        return exclusive
+    inclusive = _date(row.get("date_end") or row.get("valid_to"))
+    if not inclusive:
+        return None
+    return inclusive + timedelta(days=1) if row.get("date_end_inclusive", True) else inclusive
+
+
+def _amount(row, sum_key="sum", price_key="price", qty_key="qty"):
+    """Получить сумму напрямую либо вычислить её как точное price × qty."""
+    total = _decimal(row.get(sum_key))
+    if total is not None:
+        return total
+    price, qty = _decimal(row.get(price_key)), _decimal(row.get(qty_key))
+    return price * qty if price is not None and qty is not None else None
+
+
+def _vat_mode(value):
+    """Свести известные подписи режима НДС к gross/net/unknown."""
+    mode = str(value or "unknown").strip().lower()
+    if mode in {"with_vat", "vat_included", "inclusive", "gross", "с ндс", "включен"}:
+        return "gross"
+    if mode in {"without_vat", "vat_excluded", "exclusive", "net", "без ндс", "не включен"}:
+        return "net"
+    return "unknown"
+
+
+def _vat_rate(row):
+    """Прочитать только явно переданную ставку НДС; не выводить её по календарю."""
+    value = _decimal(row.get("vat_rate"))
+    if value is None:
+        return None
+    if value > 1:
+        value /= Decimal("100")
+    if value < 0 or value > 1:
+        return None
+    return value
+
+
+def _to_gross(amount, mode, rate):
+    """Нормализовать amount к gross, округляя по правилам денежных копеек."""
+    normalized_mode = _vat_mode(mode)
+    if amount is None:
+        return None
+    if normalized_mode == "gross":
+        return amount.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)
+    if normalized_mode == "net" and rate is not None:
+        return (amount * (Decimal("1") + rate)).quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)
+    return None
+
+
+def _vat_comparable(contract, external):
+    """Разрешить денежную сверку при одинаковом режиме или двух известных ставках."""
+    contract_mode, external_mode = _vat_mode(contract.get("vat_mode")), _vat_mode(external.get("vat_mode"))
+    if contract_mode == external_mode and contract_mode != "unknown":
+        return True
+    if contract_mode == "unknown" and external_mode == "unknown":
+        return True
+    return _vat_rate(contract) is not None and _vat_rate(external) is not None
+
+
+def _money_comparison(contract, external):
+    """Сравнить amount, price и qty с точным VAT normalization."""
+    if not _vat_comparable(contract, external):
+        return {"comparable": False, "matches": False, "reason": "VAT mode/rate is not comparable"}
+    contract_rate, external_rate = _vat_rate(contract), _vat_rate(external)
+    contract_amount = _to_gross(_amount(contract), contract.get("vat_mode"), contract_rate)
+    external_amount = _to_gross(_amount(external), external.get("vat_mode"), external_rate)
+    if contract_amount is None or external_amount is None:
+        return {"comparable": False, "matches": False, "reason": "amount is missing"}
+    contract_qty, external_qty = _decimal(contract.get("qty")), _decimal(external.get("qty"))
+    if contract_qty is not None and external_qty is not None and contract_qty != external_qty:
+        return {"comparable": True, "matches": False, "reason": "quantity differs"}
+    contract_price = _to_gross(_decimal(contract.get("price")), contract.get("vat_mode"), contract_rate)
+    external_price = _to_gross(_decimal(external.get("price")), external.get("vat_mode"), external_rate)
+    if contract_price is not None and external_price is not None:
+        if abs(contract_price - external_price) > MONEY_TOLERANCE:
+            return {"comparable": True, "matches": False, "reason": "unit price differs"}
+    if abs(contract_amount - external_amount) > MONEY_TOLERANCE:
+        return {"comparable": True, "matches": False, "reason": "sum differs"}
+    return {"comparable": True, "matches": True, "reason": ""}
+
+
+def _find_service(external, items, catalog_entries):
+    """Сопоставить external row по явному ID, SKU или семейству/anchor."""
+    explicit_id = external.get("service_item_id")
+    if explicit_id:
+        exact = [item for item in items if item.get("id") == explicit_id]
+        if exact:
+            return {"status": "MATCHED", "service_item": exact[0], "candidates": exact}
+    external_sku = re.sub(r"[^0-9A-ZА-ЯЁ-]", "", str(external.get("sku") or "").upper())
+    if external_sku:
+        sku_matches = [
+            item for item in items
+            if re.sub(r"[^0-9A-ZА-ЯЁ-]", "", str(item.get("catalog_sku") or "").upper()) == external_sku
+        ]
+        if len(sku_matches) == 1:
+            return {"status": "MATCHED", "service_item": sku_matches[0], "candidates": sku_matches}
+    raw_name = external.get("name") or external.get("service_name") or ""
+    if external_sku and catalog_entries:
+        sku_candidates = []
+        for item in items:
+            catalog_result = match_catalog_sku(
+                {**item, "charge_type": external.get("charge_type")}, catalog_entries,
+            )
+            if catalog_result.get("sku") and str(catalog_result["sku"]).upper() == external_sku:
+                sku_candidates.append(item)
+        if len(sku_candidates) == 1:
+            return {"status": "MATCHED", "service_item": sku_candidates[0], "candidates": sku_candidates}
+        if sku_candidates:
+            return {"status": "AMBIGUOUS", "service_item": None, "candidates": sku_candidates}
+    identity = analyze_service_row(raw_name, external.get("charge_type"))
+    return match_service_item(identity, items)
+
+
+def _date_comparison(interval, external):
+    """Сверить точные границы external row с интервалом договора."""
+    contract_start = _date(interval.get("valid_from"))
+    external_start = _date(external.get("date_start") or external.get("valid_from"))
+    contract_end = _date(interval.get("valid_to"))
+    external_end = _external_end(external)
+    if external_start != contract_start or external_end != contract_end:
+        return {
+            "matches": False,
+            "contract_start": contract_start.isoformat() if contract_start else None,
+            "external_start": external_start.isoformat() if external_start else None,
+            "contract_end_exclusive": contract_end.isoformat() if contract_end else None,
+            "external_end_exclusive": external_end.isoformat() if external_end else None,
+        }
+    return {"matches": True}
+
+
+def _finding(status, external=None, service_item=None, interval=None, reason="", **details):
+    """Собрать отчётный finding только из JSON-совместимых значений."""
+    return {
+        "status": status,
+        "external_row": _json_safe(external),
+        "service_item_id": service_item.get("id") if service_item else None,
+        "service_name": (interval or {}).get("raw_name") or (service_item or {}).get("canonical_name"),
+        "interval_id": (interval or {}).get("id"),
+        "reason": reason,
+        **details,
+    }
+
+
+def _json_safe(value):
+    """Сериализовать Decimal/date в точные строки для Flask JSON response."""
+    if isinstance(value, Decimal):
+        return format(value, "f")
+    if isinstance(value, (date, datetime)):
+        return value.isoformat()
+    if isinstance(value, dict):
+        return {key: _json_safe(item) for key, item in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_json_safe(item) for item in value]
+    return value
+
+
+def reconcile_dossier(dossier_id, external_rows, target_date_or_period=None,
+                      catalog_entries=None):
+    """Сверить CRM/1С строки с активными обязательствами dossier.
+
+    Входные даты `date_end` считаются включительными; `date_end_exclusive` можно
+    передать для явной полуоткрытой границы. Неоднозначные service matches и
+    открытые исключения получают NEEDS_HUMAN_DECISION. SKU UNKNOWN не меняет
+    результат сверки дат, количеств и денежных сумм.
+    """
+    dossier = db_dossiers.get_dossier(dossier_id)
+    if dossier is None:
+        return {"ok": False, "error": "dossier not found", "dossier_id": dossier_id}
+    window = _window(target_date_or_period)
+    items = db_timeline.list_service_items(dossier_id)
+    intervals = [
+        interval for interval in db_timeline.list_timeline(dossier_id)
+        if _interval_overlaps(interval, window)
+    ]
+    open_exceptions = db_exceptions.list_exceptions(dossier_id=dossier_id, status="OPEN")
+    exceptions_by_item = {}
+    for issue in open_exceptions:
+        service_item_id = issue.get("service_item_id")
+        if service_item_id:
+            exceptions_by_item.setdefault(service_item_id, []).append(issue)
+
+    findings = []
+    observed_items = set()
+    uncertain_items = set()
+    for row_index, external in enumerate(external_rows or [], start=1):
+        if not isinstance(external, dict):
+            findings.append(_finding("NEEDS_HUMAN_DECISION", reason="external row must be an object", row_index=row_index))
+            continue
+        match = _find_service(external, items, catalog_entries or [])
+        if match["status"] == "AMBIGUOUS":
+            candidate_ids = [item["id"] for item in match["candidates"]]
+            uncertain_items.update(candidate_ids)
+            findings.append(_finding(
+                "NEEDS_HUMAN_DECISION", external=external, reason="service match is ambiguous",
+                candidate_service_item_ids=candidate_ids,
+            ))
+            continue
+        service_item = match.get("service_item")
+        if service_item is None:
+            findings.append(_finding(
+                "UNCONTRACTED_BILLING", external=external,
+                reason="no matching service item exists in dossier",
+            ))
+            continue
+        observed_items.add(service_item["id"])
+        catalog_status = service_item.get("sku_status") or "UNKNOWN"
+        if catalog_entries:
+            catalog_result = match_catalog_sku(
+                {**service_item, "charge_type": external.get("charge_type")},
+                catalog_entries,
+            )
+            catalog_status = catalog_result["status"]
+        if service_item["id"] in exceptions_by_item:
+            findings.append(_finding(
+                "NEEDS_HUMAN_DECISION", external=external, service_item=service_item,
+                reason="service item has open dossier exceptions",
+                sku_status=catalog_status,
+                exception_ids=[issue["id"] for issue in exceptions_by_item[service_item["id"]]],
+            ))
+            continue
+
+        matching_intervals = [
+            interval for interval in intervals
+            if interval.get("service_item_id") == service_item["id"]
+            and (not external.get("charge_type") or interval.get("charge_type") == external.get("charge_type"))
+        ]
+        if not matching_intervals:
+            findings.append(_finding(
+                "UNCONTRACTED_BILLING", external=external, service_item=service_item,
+                reason="service item has no contractual interval in the target window",
+                sku_status=catalog_status,
+            ))
+            continue
+        if len(matching_intervals) > 1:
+            exact = [interval for interval in matching_intervals if _date_comparison(interval, external)["matches"]]
+            if len(exact) == 1:
+                matching_intervals = exact
+            else:
+                uncertain_items.add(service_item["id"])
+                findings.append(_finding(
+                    "NEEDS_HUMAN_DECISION", external=external, service_item=service_item,
+                    reason="external period overlaps multiple contractual intervals",
+                    sku_status=catalog_status,
+                    candidate_interval_ids=[interval["id"] for interval in matching_intervals],
+                ))
+                continue
+
+        interval = matching_intervals[0]
+        dates = _date_comparison(interval, external)
+        if not dates["matches"]:
+            findings.append(_finding(
+                "DATE_MISMATCH", external=external, service_item=service_item,
+                interval=interval, reason="date range differs", sku_status=catalog_status, **dates,
+            ))
+            continue
+        money = _money_comparison(interval, external)
+        if not money["comparable"]:
+            findings.append(_finding(
+                "NEEDS_HUMAN_DECISION", external=external, service_item=service_item,
+                interval=interval, reason=money["reason"], sku_status=catalog_status,
+            ))
+        elif not money["matches"]:
+            findings.append(_finding(
+                "PRICE_OR_QTY_MISMATCH", external=external, service_item=service_item,
+                interval=interval, reason=money["reason"], sku_status=catalog_status,
+            ))
+        else:
+            findings.append(_finding(
+                "MATCH", external=external, service_item=service_item, interval=interval,
+                reason="dates, quantity and amount match", sku_status=catalog_status,
+            ))
+
+    for interval in intervals:
+        service_item_id = interval.get("service_item_id")
+        if service_item_id in observed_items or service_item_id in uncertain_items:
+            continue
+        if interval.get("charge_type") == "payg":
+            continue
+        service_item = next((item for item in items if item.get("id") == service_item_id), None)
+        if service_item_id in exceptions_by_item:
+            findings.append(_finding(
+                "NEEDS_HUMAN_DECISION", service_item=service_item, interval=interval,
+                reason="service item has open dossier exceptions",
+                exception_ids=[issue["id"] for issue in exceptions_by_item[service_item_id]],
+            ))
+        else:
+            findings.append(_finding(
+                "UNBILLED_SERVICE", service_item=service_item, interval=interval,
+                reason="contractual service is active but absent from external rows",
+            ))
+
+    summary = {}
+    for finding in findings:
+        summary[finding["status"]] = summary.get(finding["status"], 0) + 1
+    return {
+        "ok": True,
+        "dossier_id": dossier_id,
+        "target": {
+            "kind": window["kind"],
+            "start": window["start"].isoformat(),
+            "end_exclusive": window["end"].isoformat(),
+        },
+        "summary": summary,
+        "findings": findings,
+    }

site/services/service_identity.py

@@ -0,0 +1,152 @@
+"""Детерминированная идентификация семейства, якорей и параметров услуги.
+
+Стабильный якорь экземпляра отделён от изменяемых параметров конфигурации:
+изменение мощности, объёма ресурса или цены не должно само по себе создавать
+новую услугу, тогда как разные стойки, облачные платформы и маршруты остаются
+разными экземплярами. Неизвестные строки сохраняются как family `other`.
+"""
+import re
+
+
+_HOMOGLYPHS = str.maketrans({
+    "A": "А", "B": "В", "C": "С", "E": "Е", "H": "Н", "K": "К",
+    "M": "М", "O": "О", "P": "Р", "T": "Т", "X": "Х", "Y": "У",
+})
+_RACK_RE = re.compile(r"\b\d[A-ZА-ЯЁ]\d[A-ZА-ЯЁ]\d{3}\b")
+_NUMBER = r"(\d+(?:[.,]\d+)?)"
+
+
+def _normalized(text):
+    """Нормализовать регистр, сохраняя различимые английские маркеры услуг."""
+    return str(text or "").upper().replace("Ё", "Е")
+
+
+def _decimal_text(value):
+    """Представить числовой фрагмент с точкой и без незначащих нулей."""
+    return value.replace(",", ".").rstrip("0").rstrip(".") if "." in value or "," in value else value
+
+
+def _first_capture(pattern, text):
+    """Извлечь первое числовое значение, сохранив точность исходной строки."""
+    match = re.search(pattern, text, re.IGNORECASE)
+    return _decimal_text(match.group(1)) if match else None
+
+
+def _service_family(text):
+    """Назначить наиболее специфичное семейство по доменным словам Nubes."""
+    rules = (
+        ("cloud_drive", r"VALO\s+CLOUD|ОБЛАЧН\w*\s+ДИСК"),
+        ("rack_cabinet", r"ШКАФ\s+(?:СЕРВЕРН\w*|\d+\s*U)"),
+        ("pdu", r"БЛОК\s+РОЗЕТОК|\bPDU\b"),
+        ("fiber_line", r"ВОЛС|ВОЛОКН|ЛИНИ\w*\s+СВЯЗИ"),
+        ("l2_channel", r"КАНАЛ\s+СВЯЗИ|\bL2\b|ТОЧКА\s*[-–—]?\s*ТОЧКА"),
+        ("network_port", r"АРЕНДА\s+ПОРТА|ПОРТ\s+В\s+КОММУТАТОРЕ|CROSS\s*[-–]?\s*CONNECT|КРОСС\s*[-–]?\s*СОЕДИН"),
+        ("ip_subnet", r"ПОДСЕТ|IPV[46]|IP\s*[-–]?\s*АДРЕС|/\s*(?:24|29|30)\b"),
+        ("internet", r"ИНТЕРНЕТ|ТРАФИК|DDOS"),
+        ("cloud_iaas", r"NEXT\s+GENERATION\s+CLOUD|ОБЛАЧН\w*|VCPU|VRAM|\bIAAS\b"),
+        ("waf", r"\bWAF\b|ЗАЩИТ\w*\s+ВЕБ"),
+        ("colocation_rack", r"СТОЙКО\s*[-–—]?\s*МЕСТ|COLOCATION|\b\d+\s*U\b"),
+    )
+    for family, pattern in rules:
+        if re.search(pattern, text, re.IGNORECASE):
+            return family
+    return "other"
+
+
+def _extract_anchors(text, family):
+    """Извлечь идентификаторы экземпляра, не перенося туда изменяемые параметры."""
+    anchors = set(_RACK_RE.findall(text))
+    if family == "fiber_line":
+        route = re.search(
+            r"(?:ОТ\s+)?(.{0,100}?)(?:\s+ДО\s+)(.{0,100})",
+            text,
+            re.IGNORECASE,
+        )
+        if route:
+            anchors.add("ROUTE:" + re.sub(r"\s+", " ", route.group(1) + "->" + route.group(2)).strip())
+    if family == "cloud_iaas":
+        platform = re.search(r"\b(?:AMD|INTEL)\s*\d+(?:[.,]\d+)?\s*ГГЦ|\bHIGHFREQ\b|\bSTD\b", text)
+        if platform:
+            anchors.add("PLATFORM:" + re.sub(r"\s+", "", platform.group(0)))
+    if family == "ip_subnet":
+        subnet = re.search(r"/\s*(\d{1,2})\b", text)
+        if subnet:
+            anchors.add("PREFIX:/" + subnet.group(1))
+    if family == "network_port":
+        speed = re.search(r"\b\d+(?:[.,]\d+)?\s*(?:ГБИТ/С|МБИТ/С|GBPS|MBPS)\b", text)
+        if speed:
+            anchors.add("SPEED:" + re.sub(r"\s+", "", speed.group(0)))
+    return "|".join(sorted(anchors))
+
+
+def _extract_config(text):
+    """Извлечь известные изменяемые численные параметры как точные строки."""
+    params = {}
+    patterns = {
+        "kw": rf"{_NUMBER}\s*КВТ\b",
+        "units_u": rf"(?:РАЗМЕР\s+СТОЙКИ\s*[:\-]?\s*)?(\d+)\s*U\b",
+        "vcpu": rf"(?:КОЛИЧЕСТВО\s*)?VCPU\s*[:\-]?\s*(\d+)",
+        "vram_gb": rf"(?:ОБЪ[ЕЕ]М\s*)?VRAM\s*[:\-]?\s*{_NUMBER}\s*ГБ",
+        "sata_gb": rf"SATA\s*[:\-]?\s*{_NUMBER}\s*ГБ",
+        "ssd_gb": rf"(?<!FAST\s)SSD\s*[:\-]?\s*{_NUMBER}\s*ГБ",
+        "fast_ssd_gb": rf"FAST\s+SSD\s*[:\-]?\s*{_NUMBER}\s*ГБ",
+        "users_count": rf"(?:ЧИСЛО\s+)?ПОЛЬЗОВАТЕЛ\w*\s*[:\-]?\s*(\d+)",
+    }
+    for key, pattern in patterns.items():
+        value = _first_capture(pattern, text)
+        if value is not None:
+            params[key] = value
+    speed = re.search(rf"{_NUMBER}\s*(МБИТ/С|ГБИТ/С|MBPS|GBPS)\b", text, re.IGNORECASE)
+    if speed:
+        unit = speed.group(2).upper()
+        params["speed_gbps" if unit in {"ГБИТ/С", "GBPS"} else "speed_mbps"] = _decimal_text(speed.group(1))
+    return params
+
+
+def analyze_service_row(raw_name, charge_type="unknown"):
+    """Вернуть семейство, anchor и извлечённые параметры для одной строки услуги."""
+    normalized = _normalized(raw_name)
+    family = _service_family(normalized)
+    return {
+        "service_family": family,
+        "instance_anchor": _extract_anchors(normalized, family),
+        "config_params": _extract_config(normalized),
+        "canonical_name": re.sub(r"\s+", " ", str(raw_name or "")).strip(),
+        "charge_type": charge_type or "unknown",
+    }
+
+
+def _anchors_compatible(left, right):
+    """Считать совместимыми одинаковые якоря либо отсутствие данных на одной стороне."""
+    left_set = {part for part in str(left or "").split("|") if part}
+    right_set = {part for part in str(right or "").split("|") if part}
+    return not left_set or not right_set or left_set == right_set
+
+
+def match_service_item(row_info, existing_service_items, llm_client=None):
+    """Сопоставить строку с существующим item без выдуманного автоматического выбора.
+
+    Если один кандидат совместим по семейству и anchor, он возвращается как
+    MATCHED. При нескольких кандидатах точное совпадение непустого anchor
+    разрешает выбор; иначе вызывающий код должен создать исключение AMBIGUOUS.
+    Параметр `llm_client` зарезервирован для будущего Entity Linker, но сейчас
+    намеренно не вызывает модель, пока это не будет отдельным проверяемым этапом.
+    """
+    del llm_client
+    same_family = [
+        item for item in existing_service_items
+        if item.get("service_family") == row_info.get("service_family")
+    ]
+    compatible = [
+        item for item in same_family
+        if _anchors_compatible(row_info.get("instance_anchor"), item.get("instance_anchor"))
+    ]
+    if len(compatible) == 1:
+        return {"status": "MATCHED", "service_item": compatible[0], "candidates": compatible}
+    row_anchor = row_info.get("instance_anchor") or ""
+    exact = [item for item in compatible if item.get("instance_anchor") == row_anchor and row_anchor]
+    if len(exact) == 1:
+        return {"status": "MATCHED", "service_item": exact[0], "candidates": exact}
+    if compatible:
+        return {"status": "AMBIGUOUS", "service_item": None, "candidates": compatible}
+    return {"status": "NOT_FOUND", "service_item": None, "candidates": []}

site/services/timeline_builder.py

@@ -0,0 +1,467 @@
+"""Построение истории услуг из паспортов документов dossier.
+
+Сервис преобразует извлечённые утверждения в стабильные ServiceItem и интервалы
+обязательств. Полный snapshot закрывает только отсутствующие recurring-услуги;
+разовые и PAYG-позиции не удаляются из истории. Дата окончания в документе
+считается включительной и переводится в исключительную границу следующего дня,
+поскольку DB-срез использует интервалы [valid_from, valid_to).
+"""
+import json
+import re
+from datetime import date, datetime, timedelta
+from decimal import Decimal, InvalidOperation
+
+from db import dossiers as db_dossiers
+from db import exceptions as db_exceptions
+from db import extractions as db_extractions
+from db import timeline as db_timeline
+from db.connection import execute, query, transaction
+from services.metrics import normalize_date
+from services.service_identity import analyze_service_row, match_service_item
+
+
+def _date(value):
+    """Нормализовать дату в ISO и отклонить несуществующие календарные даты."""
+    if not value:
+        return None
+    normalized = normalize_date(str(value).strip())
+    try:
+        return datetime.strptime(normalized, "%Y-%m-%d").date()
+    except (TypeError, ValueError):
+        return None
+
+
+def _date_text(value):
+    """Сериализовать дату для SQLite либо оставить SQL NULL."""
+    return value.isoformat() if value else None
+
+
+def _decimal_text(value):
+    """Сохранить числовое значение без float-преобразований и лишних нулей."""
+    if value is None or value == "":
+        return None
+    try:
+        number = Decimal(str(value).replace("\xa0", "").replace(" ", "").replace(",", "."))
+    except (InvalidOperation, ValueError, TypeError):
+        return None
+    if not number.is_finite():
+        return None
+    return format(number.normalize(), "f")
+
+
+def _effective_dates(row, section, passport, document):
+    """Получить valid_from/valid_to и provenance дат для одной строки.
+
+    Явная дата строки приоритетнее периода секции. При унаследованном конце
+    периода дата интерпретируется как включительная и получает границу +1 день.
+    Дата документа используется только как последнее известное начало, но не
+    маскируется под явно указанную дату услуги.
+    """
+    row_start = _date(row.get("date_start"))
+    section_start = _date(section.get("period_start"))
+    document_date = _date(passport.get("own_date") or document.get("doc_date"))
+    start = row_start or section_start or document_date
+    row_end = _date(row.get("date_end"))
+    section_end = _date(section.get("period_end"))
+    inclusive_end = row_end or section_end
+    end = inclusive_end + timedelta(days=1) if inclusive_end else None
+    if row_start:
+        date_status = row.get("date_status") or "explicit"
+        date_basis = "row.date_start"
+    elif section_start:
+        date_status = "inherited" if row.get("date_status") != "unknown" else "unknown"
+        date_basis = "section.period_start"
+    else:
+        date_status = "unknown"
+        date_basis = "document.own_date" if document_date else "missing"
+    return start, end, date_status, date_basis
+
+
+def _config_json(config_params):
+    """Детерминированно сериализовать параметры конфигурации для сравнения."""
+    return json.dumps(config_params or {}, ensure_ascii=False, sort_keys=True, default=str)
+
+
+def _row_quote(row):
+    """Собрать короткую аудируемую цитату исходной строки."""
+    return str(row.get("raw_name") or "")[:1000]
+
+
+def _add_lineage_exception(dossier_id, batch_id, document_id, title,
+                            quote="", service_item_id=None, options=None):
+    """Создать исключение сопоставления один раз на документ и заголовок."""
+    existing = db_exceptions.list_exceptions(
+        dossier_id=dossier_id, status="OPEN",
+    )
+    if any(
+        item.get("document_id") == document_id
+        and item.get("stage") == "lineage_matching"
+        and item.get("title") == title
+        for item in existing
+    ):
+        return
+    db_exceptions.add_exception(
+        batch_id=batch_id, dossier_id=dossier_id, document_id=document_id,
+        service_item_id=service_item_id, severity="AMBIGUOUS",
+        stage="lineage_matching", title=title, source_quote=quote,
+        options=options or [],
+    )
+
+
+def _passport_for(document):
+    """Вернуть паспорт документа или пустую форму для неполных legacy-досье."""
+    passport = db_extractions.get_by_document(document["document_id"])
+    return passport or {
+        "document_id": document["document_id"], "doc_class": "unknown",
+        "intent_mode": "no_spec", "own_date": None, "sections": [],
+        "math_validation": {},
+    }
+
+
+def _ordered_documents(dossier_id):
+    """Упорядочить документы по роли, затем по дате и номеру допсоглашения."""
+    documents = db_dossiers.list_dossier_documents(dossier_id)
+    role_order = {"frame_contract": 0, "base_spec": 1, "specification": 2, "supplement": 3}
+
+    def key(document):
+        passport = _passport_for(document)
+        role = document.get("role") or "document"
+        own_number = str(passport.get("own_number") or "")
+        digits = re.sub(r"\D", "", own_number)
+        return (
+            role_order.get(role, 4),
+            int(digits) if digits else 0,
+            passport.get("own_date") or document.get("doc_date") or "9999",
+            document["document_id"],
+        )
+
+    return sorted(documents, key=key)
+
+
+def _interval_content_matches(interval, row, identity, section):
+    """Сравнить изменяемые утверждения, не используя их для entity identity."""
+    return (
+        interval.get("charge_type") == (row.get("charge_type") or section.get("charge_type") or "unknown")
+        and _decimal_text(interval.get("price")) == _decimal_text(row.get("price"))
+        and _decimal_text(interval.get("qty")) == _decimal_text(row.get("qty"))
+        and _decimal_text(interval.get("sum")) == _decimal_text(row.get("sum"))
+        and interval.get("config_params_json") == _config_json(identity.get("config_params"))
+        and interval.get("vat_mode", "vat_unknown") == (section.get("vat_mode") or "vat_unknown")
+    )
+
+
+def _close_open_interval(interval, close_date, document_id):
+    """Закрыть открытый интервал только если новая граница после его начала."""
+    start = _date(interval.get("valid_from"))
+    if close_date and start and close_date > start:
+        db_timeline.close_interval(interval["id"], close_date.isoformat(), document_id)
+        return True
+    return False
+
+
+def _append_interval(dossier_id, service_item, row, section, document,
+                     passport, start, end, date_status, date_basis, status):
+    """Записать один интервал с точными десятичными строками и provenance."""
+    identity = analyze_service_row(row.get("raw_name"), row.get("charge_type") or section.get("charge_type"))
+    return db_timeline.insert_interval({
+        "dossier_id": dossier_id,
+        "service_item_id": service_item["id"],
+        "charge_type": row.get("charge_type") or section.get("charge_type") or "unknown",
+        "valid_from": _date_text(start),
+        "valid_to": _date_text(end),
+        "date_status": date_status,
+        "date_basis": date_basis,
+        "raw_name": row.get("raw_name") or identity["canonical_name"] or "",
+        "config_params_json": _config_json(identity["config_params"]),
+        "price": _decimal_text(row.get("price")),
+        "qty": _decimal_text(row.get("qty")),
+        "sum": _decimal_text(row.get("sum")),
+        "vat_mode": section.get("vat_mode") or "vat_unknown",
+        "vat_rate": _decimal_text(section.get("vat_rate")),
+        "source_document_id": document["document_id"],
+        "table_ref": section.get("table_ref") or "",
+        "quote": _row_quote(row),
+        "status": status,
+    })
+
+
+def _resolved_option_for(document_id, row, resolved_exceptions):
+    """Найти сохранённый выбор для точной строки и документа lineage исключения."""
+    quote = _row_quote(row)
+    for issue in resolved_exceptions:
+        if (
+            issue.get("stage") == "lineage_matching"
+            and issue.get("document_id") == document_id
+            and issue.get("source_quote") == quote
+        ):
+            resolution = issue.get("resolution")
+            if isinstance(resolution, dict):
+                option = resolution.get("selected_option")
+                return option if isinstance(option, dict) else None
+    return None
+
+
+def _reusable_service_item(identity, service_items):
+    """Найти прежнюю устойчивую identity, не включая будущие строки в matching."""
+    family = identity.get("service_family")
+    anchor = identity.get("instance_anchor") or ""
+    candidates = [item for item in service_items if item.get("service_family") == family]
+    if anchor:
+        candidates = [item for item in candidates if item.get("instance_anchor", "") == anchor]
+    else:
+        candidates = [
+            item for item in candidates
+            if item.get("canonical_name") == identity.get("canonical_name")
+        ]
+    return candidates[0] if len(candidates) == 1 else None
+
+
+def _process_document(dossier, document, passport, items, counts,
+                      resolved_exceptions=None, reusable_items=None):
+    """Reconcile every section of one passport against current dossier services."""
+    doc_id = document["document_id"]
+    mode = passport.get("intent_mode")
+    if mode == "no_spec" or passport.get("doc_class") not in {"specification", "supplement"}:
+        return
+    sections = [section for section in passport.get("sections") or [] if isinstance(section, dict)]
+    sections.sort(key=lambda section: _date(section.get("period_start")) or date.max)
+    seen_recurring = set()
+    effective_dates = []
+    math_data = passport.get("math_validation") or {}
+    extraction_status = passport.get("status") or "UNKNOWN"
+    snapshot_is_complete = mode == "full_snapshot" and extraction_status not in {"CONFLICT", "FAILED"}
+
+    for section in sections:
+        rows = [row for row in section.get("rows") or [] if isinstance(row, dict)]
+        for row in rows:
+            action = (row.get("delta_action") or ("snapshot" if mode == "full_snapshot" else "unknown")).lower()
+            charge_type = row.get("charge_type") or section.get("charge_type") or "unknown"
+            if mode == "delta_amendment" and action in {"snapshot", "unknown", "unresolved"}:
+                _add_lineage_exception(
+                    dossier["id"], dossier.get("batch_id"), doc_id,
+                    "UNRESOLVED_DELTA_ACTION", _row_quote(row),
+                )
+                counts["ambiguous"] += 1
+                continue
+            start, end, date_status, date_basis = _effective_dates(row, section, passport, document)
+            if start:
+                effective_dates.append(start)
+            identity = analyze_service_row(row.get("raw_name"), charge_type)
+            match = match_service_item(identity, items)
+            if match["status"] == "AMBIGUOUS":
+                selected_option = _resolved_option_for(
+                    doc_id, row, resolved_exceptions or [],
+                )
+                if selected_option:
+                    selected_id = selected_option.get("service_item_id")
+                    candidates = [item for item in match["candidates"] if item.get("id") == selected_id]
+                    if not candidates:
+                        candidates = [
+                            item for item in match["candidates"]
+                            if item.get("service_family") == selected_option.get("service_family")
+                            and item.get("instance_anchor", "") == selected_option.get("instance_anchor", "")
+                        ]
+                    if len(candidates) == 1:
+                        match = {
+                            "status": "MATCHED", "service_item": candidates[0],
+                            "candidates": candidates,
+                        }
+            if match["status"] == "AMBIGUOUS":
+                _add_lineage_exception(
+                    dossier["id"], dossier.get("batch_id"), doc_id,
+                    "AMBIGUOUS_SERVICE_ITEM", _row_quote(row),
+                    options=[{
+                        "service_item_id": item["id"],
+                        "service_family": item.get("service_family"),
+                        "instance_anchor": item.get("instance_anchor", ""),
+                        "canonical_name": item.get("canonical_name"),
+                    } for item in match["candidates"]],
+                )
+                counts["ambiguous"] += 1
+                continue
+            service_item = match["service_item"]
+            if service_item is None:
+                service_item = _reusable_service_item(identity, reusable_items or [])
+                if service_item is None:
+                    service_item = db_timeline.insert_service_item(
+                        dossier_id=dossier["id"],
+                        service_family=identity["service_family"],
+                        canonical_name=identity["canonical_name"] or "(empty)",
+                        instance_anchor=identity["instance_anchor"],
+                    )
+                items.append(service_item)
+
+            open_intervals = [
+                interval for interval in db_timeline.list_open_intervals(dossier["id"], service_item["id"])
+                if interval.get("charge_type") == charge_type
+            ]
+            history = [
+                interval for interval in db_timeline.list_timeline(dossier["id"])
+                if interval.get("service_item_id") == service_item["id"]
+                and interval.get("charge_type") == charge_type
+            ]
+            if charge_type == "recurring":
+                seen_recurring.add(service_item["id"])
+
+            if action in {"delete", "terminate"}:
+                if not start:
+                    _add_lineage_exception(
+                        dossier["id"], dossier.get("batch_id"), doc_id,
+                        "TERMINATION_DATE_UNKNOWN", _row_quote(row), service_item["id"],
+                    )
+                    counts["ambiguous"] += 1
+                elif not open_intervals:
+                    _add_lineage_exception(
+                        dossier["id"], dossier.get("batch_id"), doc_id,
+                        "TERMINATION_TARGET_NOT_OPEN", _row_quote(row), service_item["id"],
+                    )
+                    counts["ambiguous"] += 1
+                else:
+                    for interval in open_intervals:
+                        if _close_open_interval(interval, start, doc_id):
+                            counts["closed"] += 1
+                continue
+
+            if not start:
+                _add_lineage_exception(
+                    dossier["id"], dossier.get("batch_id"), doc_id,
+                    "SERVICE_START_DATE_UNKNOWN", _row_quote(row), service_item["id"],
+                )
+                date_status = "unknown"
+                interval_status = "WARNING"
+                counts["ambiguous"] += 1
+            else:
+                interval_status = "CONFLICT" if math_data.get("status") == "CONFLICT" else "VERIFIED"
+
+            previous = open_intervals[-1] if open_intervals else None
+            if previous and _interval_content_matches(previous, row, identity, section):
+                previous_start = _date(previous.get("valid_from"))
+                if previous_start == start or start is None:
+                    new_end = end or previous.get("valid_to")
+                    db_timeline.update_interval(previous["id"], {
+                        "valid_to": _date_text(end) if end else new_end,
+                        "date_status": date_status,
+                    })
+                    counts["unchanged"] += 1
+                    continue
+
+            if not previous and history and start:
+                latest = history[-1]
+                if (
+                    latest.get("valid_to") == start.isoformat()
+                    and _interval_content_matches(latest, row, identity, section)
+                ):
+                    db_timeline.update_interval(latest["id"], {
+                        "valid_to": _date_text(end),
+                        "closed_by_document_id": None,
+                    })
+                    counts["unchanged"] += 1
+                    continue
+
+            if previous and start and _date(previous.get("valid_from")) == start:
+                db_timeline.update_interval(previous["id"], {
+                    "valid_to": _date_text(end),
+                    "raw_name": row.get("raw_name") or "",
+                    "config_params_json": _config_json(identity["config_params"]),
+                    "price": _decimal_text(row.get("price")),
+                    "qty": _decimal_text(row.get("qty")),
+                    "sum": _decimal_text(row.get("sum")),
+                    "vat_mode": section.get("vat_mode") or "vat_unknown",
+                    "vat_rate": _decimal_text(section.get("vat_rate")),
+                    "source_document_id": doc_id,
+                    "closed_by_document_id": None,
+                    "table_ref": section.get("table_ref") or "",
+                    "quote": _row_quote(row),
+                    "status": interval_status,
+                })
+                counts["updated"] += 1
+                continue
+
+            for interval in open_intervals:
+                if start and _close_open_interval(interval, start, doc_id):
+                    counts["closed"] += 1
+            _append_interval(
+                dossier["id"], service_item, row, section, document, passport,
+                start, end, date_status, date_basis, interval_status,
+            )
+            counts["created"] += 1
+
+    if snapshot_is_complete and sections and effective_dates:
+        snapshot_date = min(effective_dates)
+        for service_item in items:
+            if service_item["id"] in seen_recurring:
+                continue
+            for interval in db_timeline.list_open_intervals(dossier["id"], service_item["id"]):
+                if interval.get("charge_type") == "recurring" and _close_open_interval(interval, snapshot_date, doc_id):
+                    counts["closed"] += 1
+
+
+def build_dossier_timeline(dossier_id, contract_id=None):
+    """Пересобрать таймлайн dossier из сохранённых паспортов без повторного вызова LLM.
+
+    Для legacy-синхронизации caller должен передать `contract_id`: в схеме нет
+    автоматической связи `contract_dossiers.id` с `contracts.id`, поэтому сервис
+    не выводит её из похожих номеров или имён контрагентов.
+    """
+    dossier = db_dossiers.get_dossier(dossier_id)
+    if dossier is None:
+        return {"ok": False, "error": "dossier not found", "dossier_id": dossier_id}
+    documents = _ordered_documents(dossier_id)
+    resolved_exceptions = db_exceptions.list_exceptions(
+        dossier_id=dossier_id, status="RESOLVED",
+    )
+    items = []
+    reusable_items = db_timeline.list_service_items(dossier_id)
+    counts = {"created": 0, "updated": 0, "unchanged": 0, "closed": 0, "ambiguous": 0}
+
+    with transaction():
+        execute(
+            """DELETE FROM exceptions_queue
+               WHERE dossier_id = %s AND stage = 'lineage_matching' AND status = 'OPEN'""",
+            (dossier_id,),
+        )
+        execute("DELETE FROM timeline_intervals WHERE dossier_id = %s", (dossier_id,))
+        for document in documents:
+            passport = _passport_for(document)
+            _process_document(
+                dossier, document, passport, items, counts,
+                resolved_exceptions, reusable_items,
+            )
+
+    if contract_id:
+        sync_legacy_spec_current(dossier_id, contract_id)
+    return {
+        "ok": True,
+        "dossier_id": dossier_id,
+        "counts": counts,
+        "timeline": db_timeline.list_timeline(dossier_id),
+        "exceptions": db_exceptions.list_exceptions(dossier_id=dossier_id, status="OPEN"),
+    }
+
+
+def sync_legacy_spec_current(dossier_id, contract_id, as_of_date=None):
+    """Материализовать срез таймлайна в legacy spec_current по явному contract_id."""
+    current_date = _date(as_of_date) or date.today()
+    active_intervals = db_timeline.list_timeline(dossier_id, current_date.isoformat())
+    with transaction():
+        execute("DELETE FROM spec_current WHERE contract_id = %s", (contract_id,))
+        for interval in active_intervals:
+            if interval.get("valid_from") is None:
+                continue
+            item = db_timeline.get_service_item(interval["service_item_id"])
+            if not item:
+                continue
+            name_hash = __import__("hashlib").sha256(
+                f"{item['canonical_name'].strip().lower()}|{interval['valid_from']}".encode(),
+            ).hexdigest()[:16]
+            execute(
+                """INSERT INTO spec_current
+                   (id, contract_id, name_hash, name, price, qty, sum, date_start)
+                   VALUES (%s, %s, %s, %s, %s, %s, %s, %s)""",
+                (
+                    interval["id"], contract_id, name_hash, interval["raw_name"],
+                    interval.get("price"), interval.get("qty"), interval.get("sum"),
+                    interval["valid_from"],
+                ),
+            )
+    return len(active_intervals)

site/static/app.js

@@ -14,7 +14,7 @@ var VM_UPLOAD_URL = 'https://contracts-upload.sinefo.ru/contracts-upload/';
 // Прогрев приложения при загрузке страницы.
 // ⚠️ Без прогрева первый POST может упасть с ERR_CONNECTION_RESET (MTU/Geneve)
 (async function init() {
-  try { await fetch('/health'); } catch(e) { /* ignore */ }
+  try { await fetch('/health'); } catch (e) { /* ignore */ }
 })();
 
 /**
@@ -31,15 +31,15 @@ filePicker = window.FilePicker.initFilePicker({
   allowedExt: ['.pdf', '.doc', '.docx'],
   limits: FILE_PICKER_LIMITS,
   onChange: onPickerFilesChange,
-  onError: function(error, file) {
+  onError: function (error, file) {
     filePicker.setMessage('Ошибка чтения ' + file.name + ': ' + error.message);
   },
-  onDetails: function(id) {
-    var index = state.files.findIndex(function(file) { return file.pickerId === id; });
+  onDetails: function (id) {
+    var index = state.files.findIndex(function (file) { return file.pickerId === id; });
     if (index >= 0) toggleClassifyDetail(index);
   },
-  onText: function(id) {
-    var index = state.files.findIndex(function(file) { return file.pickerId === id; });
+  onText: function (id) {
+    var index = state.files.findIndex(function (file) { return file.pickerId === id; });
     if (index >= 0) showText(index);
   }
 });
@@ -59,10 +59,10 @@ function renderStepper(state) {
   var steps = state.ui.steps;
   var map = { upload: 'stepUpload', classify: 'stepClassify', groups: 'stepGroups', compare: 'stepCompare' };
   var colors = { '○': 'var(--topbar-muted)', '⏳': 'var(--topbar-active)', '✓': 'var(--topbar-done)' };
-  Object.keys(map).forEach(function(key) {
+  Object.keys(map).forEach(function (key) {
     var el = document.getElementById(map[key]);
     if (el) {
-      el.textContent = steps[key] + ' ' + {upload:'Загрузка',classify:'Классификация',groups:'Группировка',compare:'Сравнение'}[key];
+      el.textContent = steps[key] + ' ' + { upload: 'Загрузка', classify: 'Классификация', groups: 'Группировка', compare: 'Сравнение' }[key];
       el.style.color = colors[steps[key]] || 'var(--muted)';
     }
   });
@@ -92,10 +92,10 @@ function stepActive(id) {
  * Мутирует state.ui.steps, вызывает renderStepper.
  */
 function resetStepper(fromId) {
-  var order = ['stepUpload','stepClassify','stepGroups','stepCompare'];
-  var keys  = ['upload','classify','groups','compare'];
+  var order = ['stepUpload', 'stepClassify', 'stepGroups', 'stepCompare'];
+  var keys = ['upload', 'classify', 'groups', 'compare'];
   var reset = false;
-  order.forEach(function(id, i) {
+  order.forEach(function (id, i) {
     if (id === fromId) reset = true;
     if (reset) { state.ui.steps[keys[i]] = '○'; }
   });
@@ -133,7 +133,7 @@ async function runClassify() {
   stepActive('stepClassify');
 
   var start = Date.now();
-  var timer = setInterval(function() {
+  var timer = setInterval(function () {
     btn.innerHTML = '<span class="spinner"></span> Классификация... ' + Math.round((Date.now() - start) / 1000) + 'с';
   }, 1000);
 
@@ -141,14 +141,41 @@ async function runClassify() {
   var classifyDone = false;
 
   // Poll progress каждые 2с — работает и для sync, и для async classify
-  var pollTimer = setInterval(async function() {
+  var pollTimer = setInterval(async function () {
     try {
-      var r = await fetch(VM_API + '/api/batch-progress?batch=' + state.batchId);
+      var progressUrl = state.v3Mode
+        ? '/api/classify-progress?batch=' + encodeURIComponent(state.batchId)
+        : VM_API + '/api/batch-progress?batch=' + encodeURIComponent(state.batchId);
+      var r = await fetch(progressUrl);
       var d = await r.json();
+      if (state.v3Mode && d.v3) {
+        var completed = d.done || 0;
+        var total = d.total || 0;
+        btn.innerHTML = '<span class="spinner"></span> Извлечение... ' + completed + '/' + total +
+          ' (' + Math.round((Date.now() - start) / 1000) + 'с)';
+        if (d.finished) {
+          clearInterval(pollTimer);
+          clearInterval(timer);
+          classifyDone = true;
+          if (d.state === 'failed' || d.state === 'interrupted') {
+            btn.innerHTML = '✗ ' + escHtml(d.error || 'ошибка извлечения');
+            btn.disabled = false;
+            return;
+          }
+          // Partial jobs are terminal and usable, but must not look like full success.
+          var failed = d.failed || 0;
+          btn.innerHTML = (failed ? '⚠ ' : '✓ ') + completed + '/' + total + ' документов извлечено'
+            + (failed ? ' | ошибок: ' + failed : '');
+          stepDone('stepClassify');
+          stepActive('stepGroups');
+          loadGroupsAction();
+        }
+        return;
+      }
       if (d.ok && d.counts) {
         var done = (d.counts.classified || 0)
-                 + (d.counts.failed || 0)
-                 + (d.counts.garbage || 0);
+          + (d.counts.failed || 0)
+          + (d.counts.garbage || 0);
         var total = d.total || 0;
         if (total > 0) totalFiles = total;
         btn.innerHTML = '<span class="spinner"></span> Классификация... ' + done + '/' + total + ' (' + Math.round((Date.now() - start) / 1000) + 'с)';
@@ -162,26 +189,35 @@ async function runClassify() {
           loadGroupsAction();
         }
       }
-    } catch(e) {}
+    } catch (e) { }
   }, 2000);
 
   try {
     var resp = await fetch(VM_API + '/api/classify-batch', {
       method: 'POST',
-      headers: {'Content-Type': 'application/json'},
-      body: JSON.stringify({batch_id: state.batchId})
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ batch_id: state.batchId })
     });
     var data = await resp.json();
+    state.v3Mode = Boolean(data.v3);
+    if (data.v3) state.v3Summary = data;
 
     // sync classify: ответ уже содержит done
     if (data.ok && data.done !== undefined) {
       clearInterval(timer);
       clearInterval(pollTimer);
       classifyDone = true;
-      var summary = data.summary || (data.done + '/' + data.total + ' классифицировано');
+      var summary = data.v3
+        ? data.done + '/' + data.total + ' извлечено; ' + (data.dossier_count || 0) + ' досье'
+        : data.summary || (data.done + '/' + data.total + ' классифицировано');
+      if (data.v3 && data.ignored_accounting) summary += ' | отсеяно: ' + data.ignored_accounting;
+      if (data.v3) summary += ' | арифметика: ' + data.math_checks_passed;
+      if (data.v3 && data.failed) summary += ' | ошибок: ' + data.failed;
       if (data.garbage) summary += ' | 🗑️ ' + data.garbage + ' мусор';
-      if (data.types) summary += ' | ' + Object.entries(data.types).map(function(e){return e[1]+'×'+e[0];}).join(', ');
-      btn.innerHTML = '✓ ' + summary + ' (' + Math.round((Date.now() - start) / 1000) + 'с)';
+      if (data.types) summary += ' | ' + Object.entries(data.types).map(function (e) { return e[1] + '×' + e[0]; }).join(', ');
+      // Continue to dossier rendering for partial success while keeping its warning visible.
+      btn.innerHTML = (data.v3 && data.partial ? '⚠ ' : '✓ ') + summary
+        + ' (' + Math.round((Date.now() - start) / 1000) + 'с)';
       stepDone('stepClassify');
       stepActive('stepGroups');
       loadGroupsAction();
@@ -200,7 +236,7 @@ async function runClassify() {
       btn.innerHTML = '✗ ' + (data.error || 'ошибка');
       btn.disabled = false;
     }
-  } catch(e) {
+  } catch (e) {
     if (!classifyDone) {
       clearInterval(timer);
       clearInterval(pollTimer);
@@ -215,7 +251,7 @@ async function runClassify() {
 // Текущий активный EventSource (только один одновременно)
 // state._activeCompare — теперь в state.js (Фаза 0: state + render)
 
-window.runCompareForGroup = async function(gi, btn) {
+window.runCompareForGroup = async function (gi, btn) {
   var group = state.groups[gi];
   if (!group || group.contract_number === '__unresolved__') return;
 
@@ -234,17 +270,17 @@ window.runCompareForGroup = async function(gi, btn) {
   btn.disabled = true;
   btn.innerHTML = '<span class="spinner"></span> Применяю...';
   // Заблокировать ВСЕ кнопки сравнения пока идёт процесс
-  document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = true; });
+  document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = true; });
 
   var ar = await fetch(VM_API + '/api/apply-groups', {
     method: 'POST',
-    headers: {'Content-Type': 'application/json'},
-    body: JSON.stringify({batch_id: state.batchId, groups: [group]})
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ batch_id: state.batchId, groups: [group] })
   });
   var ad = await ar.json();
   if (!ad.ok || !ad.contract_ids || !ad.contract_ids.length) {
     btn.innerHTML = '✗ ошибка'; btn.disabled = false;
-    document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = false; });
+    document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = false; });
     return;
   }
 
@@ -275,7 +311,7 @@ window.runCompareForGroup = async function(gi, btn) {
     cmpBody,
     statusEl,
     {
-      onDone: function(d, sections) {
+      onDone: function (d, sections) {
         state._activeCompare.timer = null;
         state._activeCompare.es = null;
         group.compare = {
@@ -285,23 +321,23 @@ window.runCompareForGroup = async function(gi, btn) {
           sections: sections
         };
         renderGroups(state);
-        document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = false; });
+        document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = false; });
         stepDone('stepCompare');
       },
-      onError: function(d) {
+      onError: function (d) {
         state._activeCompare.timer = null;
         state._activeCompare.es = null;
         group.compare = null;
         cmpBody.innerHTML += '<div style="color:var(--destructive);">✗ ' + escHtml(d.message) + '</div>';
         btn.innerHTML = '✗'; btn.disabled = false;
-        document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = false; });
+        document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = false; });
       },
-      onConnectionError: function() {
+      onConnectionError: function () {
         state._activeCompare.timer = null;
         state._activeCompare.es = null;
         group.compare = null;
         btn.innerHTML = '✗ соединение'; btn.disabled = false;
-        document.querySelectorAll('.cmp-btn').forEach(function(b) { b.disabled = false; });
+        document.querySelectorAll('.cmp-btn').forEach(function (b) { b.disabled = false; });
       }
     }
   );
@@ -312,7 +348,7 @@ window.runCompareForGroup = async function(gi, btn) {
 // uploadFile → files.js (Фаза 1)
 
 // ── LLM: Общее сравнение (Event Sourcing) ────────────────────────
-document.getElementById('llmBtn').addEventListener('click', function() {
+document.getElementById('llmBtn').addEventListener('click', function () {
   if (!state.contractId) return;
   var btn = this;
   btn.disabled = true;
@@ -325,13 +361,13 @@ document.getElementById('llmBtn').addEventListener('click', function() {
   compareBody.innerHTML = '';
   compareStatus.textContent = '⏳ 0.0с';
 
-  var order = state.files.map(function(f) { return f.supp_id; }).filter(Boolean).join(',');
+  var order = state.files.map(function (f) { return f.supp_id; }).filter(Boolean).join(',');
   var url = '/process-v2?contract_id=' + state.contractId +
     (order ? '&order=' + encodeURIComponent(order) : '');
 
   // Фаза 3: унифицированный SSE через startCompareSSE (compare.js)
   startCompareSSE(url, compareBody, compareStatus, {
-    onDone: function(d) {
+    onDone: function (d) {
       btn.innerHTML = '<i data-lucide="check" style="width:16px;height:16px;"></i> ✓ Готово';
       stepDone('stepCompare');
       if (d.total_unresolved > 0) {
@@ -340,12 +376,12 @@ document.getElementById('llmBtn').addEventListener('click', function() {
       document.getElementById('chatCard').style.display = 'block';
       lucide.createIcons();
     },
-    onError: function(d) {
+    onError: function (d) {
       compareBody.innerHTML += '<div style="color:var(--destructive);margin-top:8px;">✗ ' + escHtml(d.message) + '</div>';
       btn.innerHTML = '<i data-lucide="scale" style="width:16px;height:16px;"></i> Общее сравнение';
       btn.disabled = false;
     },
-    onConnectionError: function() {
+    onConnectionError: function () {
       if (compareBody.innerHTML === '') {
         compareBody.innerHTML = '<div style="color:var(--destructive);">✗ Ошибка соединения</div>';
       }
@@ -359,7 +395,7 @@ document.getElementById('llmBtn').addEventListener('click', function() {
 // refreshSupps → files.js (Фаза 1)
 
 // ── Чат ──────────────────────────────────────────────────────
-document.getElementById('chatSend').addEventListener('click', function() {
+document.getElementById('chatSend').addEventListener('click', function () {
   var input = document.getElementById('chatInput');
   var q = input.value.trim();
   if (!q || !state.contractId) return;
@@ -373,21 +409,21 @@ document.getElementById('chatSend').addEventListener('click', function() {
   var fd = new FormData();
   fd.append('question', q);
   fetch('/chat.cfm?contract_id=' + state.contractId, { method: 'POST', body: fd })
-    .then(function(r) { return r.json(); })
-    .then(function(d) {
+    .then(function (r) { return r.json(); })
+    .then(function (d) {
       msgs.lastChild.remove();
       msgs.innerHTML += '<div style="margin-top:2px;"><strong>Ответ:</strong> ' + (d.OK ? d.ANSWER : (d.ERROR || '—')) + '</div>';
       input.disabled = false;
       document.getElementById('chatSend').disabled = false;
       input.focus();
-    }).catch(function() {
+    }).catch(function () {
       msgs.lastChild.remove();
       msgs.innerHTML += '<div style="color:var(--destructive);margin-top:2px;">Ошибка сети</div>';
       input.disabled = false;
       document.getElementById('chatSend').disabled = false;
     });
 });
-document.getElementById('chatInput').addEventListener('keydown', function(e) {
+document.getElementById('chatInput').addEventListener('keydown', function (e) {
   if (e.key === 'Enter') document.getElementById('chatSend').click();
 });
 
@@ -423,24 +459,24 @@ async function showText(i) {
 
       // Общий textify для обеих панелей
       var textLines = [];
-      elements.forEach(function(el) {
+      elements.forEach(function (el) {
         var etype = el.type || el.TYPE || '?';
         if (etype === 'paragraph') {
           textLines.push(el.text || el.TEXT || '');
         } else if (etype === 'table') {
           var rows = el.rows || el.ROWS || [];
           if (rows.length > 0) {
-            var ncols = (rows[0]||[]).length;
+            var ncols = (rows[0] || []).length;
             textLines.push('--- Таблица ' + rows.length + '×' + ncols + ' ---');
-            rows.forEach(function(row) {
-              var cells = (row||[]).map(function(c){ return String(c||'').replace(/\n/g,' ').replace(/\|/g,'\\|'); });
+            rows.forEach(function (row) {
+              var cells = (row || []).map(function (c) { return String(c || '').replace(/\n/g, ' ').replace(/\|/g, '\\|'); });
               textLines.push('| ' + cells.join(' | ') + ' |');
             });
             textLines.push('');
           }
         }
       });
-      var textify = textLines.join('\n').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+      var textify = textLines.join('\n').replace(/</g, '&lt;').replace(/>/g, '&gt;');
 
       // Левая: сырой текст (textify)
       leftHtml = '<pre>' + textify + '</pre>';
@@ -450,7 +486,7 @@ async function showText(i) {
       rightHtml += '<div class="kv"><span class="k">Изменён</span><span class="v">' + formatDate(f.lastModified) + '</span></div>';
       if (f.parseInfo) {
         var d = f.parseInfo;
-        rightHtml += '<div class="kv"><span class="k">Элементов</span><span class="v">' + (d.element_count||0) + '</span></div>';
+        rightHtml += '<div class="kv"><span class="k">Элементов</span><span class="v">' + (d.element_count || 0) + '</span></div>';
         if (d.error) rightHtml += '<div style="margin-top:8px;color:var(--destructive);">' + d.error + '</div>';
       }
       rightHtml += '<div style="margin-top:10px;font-weight:600;font-size:12px;">Все элементы:</div>';
@@ -468,13 +504,13 @@ async function showText(i) {
         if (qData.doc_date) rightHtml += '<div class="kv"><span class="k">Дата</span><span class="v">' + escHtml(qData.doc_date) + '</span></div>';
         if (qData.counterparty) rightHtml += '<div class="kv"><span class="k">Контрагент</span><span class="v">' + escHtml(qData.counterparty) + '</span></div>';
         if (qData.classify_raw) {
-          var rawId = 'raw_' + docId.replace(/-/g,'');
+          var rawId = 'raw_' + docId.replace(/-/g, '');
           rightHtml += '<details style="margin-top:8px;"><summary style="cursor:pointer;font-size:11px;color:var(--brand-primary);">Сырой ответ LLM</summary>';
           rightHtml += '<pre style="font-size:10px;max-height:200px;overflow:auto;background:var(--brand-grey-light);padding:6px;border-radius:4px;margin-top:4px;">' + escHtml(qData.classify_raw) + '</pre>';
           rightHtml += '</details>';
         }
       }
-    } catch(e) {
+    } catch (e) {
       leftHtml = '<span style="color:var(--destructive);">Ошибка</span>';
     }
   } else {
@@ -504,11 +540,11 @@ var promptNotes = document.getElementById('promptNotes');
 //   extract  — извлечение строк из первого документа (базовый договор)
 //   diff     — сравнение допсоглашений (ADD/UPDATE/DELETE)
 //   classify — классификация документа (тип, номер, контрагент, дата)
-var activePromptId = {extract: '', diff: '', classify: ''};
+var activePromptId = { extract: '', diff: '', classify: '' };
 
 function openPromptModal(role) {
   promptRole = role;
-  var labels = {extract: 'Извлечение', diff: 'Сравнение', classify: 'Классификация'};
+  var labels = { extract: 'Извлечение', diff: 'Сравнение', classify: 'Классификация' };
   promptModalTitle.textContent = 'Промпт: ' + (labels[role] || role);
   document.getElementById('promptTabExtract').style.background = role === 'extract' ? 'var(--brand-primary)' : '';
   document.getElementById('promptTabExtract').style.color = role === 'extract' ? '#fff' : '';
@@ -530,26 +566,26 @@ function closePromptModal(e) {
 
 function loadPrompt(role) {
   fetch(VM_API + '/api/prompts?role=' + role)
-    .then(function(r) { return r.json(); })
-    .then(function(d) {
+    .then(function (r) { return r.json(); })
+    .then(function (d) {
       if (d.ok) {
         promptEditor.value = d.body || '';
         activePromptId[role] = d.id || '';
         loadHistory(role);
       }
     })
-    .catch(function(e) { console.error('loadPrompt:', e); });
+    .catch(function (e) { console.error('loadPrompt:', e); });
 }
 
 function loadHistory(role) {
   fetch(VM_API + '/api/prompts/list?role=' + role)
-    .then(function(r) { return r.json(); })
-    .then(function(d) {
+    .then(function (r) { return r.json(); })
+    .then(function (d) {
       if (!d.ok || !d.versions || !d.versions.length) {
         promptHistory.innerHTML = '<div style="color:var(--muted);">нет версий</div>';
         return;
       }
-      promptHistory.innerHTML = d.versions.map(function(v) {
+      promptHistory.innerHTML = d.versions.map(function (v) {
         var isActive = v.is_active ? '<span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:var(--green);margin-right:4px;" title="активная"></span> ' : '';
         var style = v.is_active ? 'font-weight:600;' : '';
         return '<div style="display:flex;align-items:center;padding:4px 0;border-bottom:1px solid var(--brand-gray);font-size:11px;' + style + '">'
@@ -561,17 +597,17 @@ function loadHistory(role) {
           + '</div>';
       }).join('');
     })
-    .catch(function(e) { console.error('loadHistory:', e); });
+    .catch(function (e) { console.error('loadHistory:', e); });
 }
 
 function activatePrompt(id) {
   fetch(VM_API + '/api/prompts/activate', {
     method: 'POST',
-    headers: {'Content-Type': 'application/json'},
-    body: JSON.stringify({id: id})
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ id: id })
   })
-    .then(function(r) { return r.json(); })
-    .then(function(d) {
+    .then(function (r) { return r.json(); })
+    .then(function (d) {
       if (d.ok) { loadPrompt(promptRole); promptStatus.textContent = '✓ активирован'; }
       else { promptStatus.textContent = '✕ ' + (d.error || 'ошибка'); }
     });
@@ -581,30 +617,30 @@ function deletePrompt(id) {
   if (!confirm('Удалить эту версию промпта?')) return;
   fetch(VM_API + '/api/prompts/delete', {
     method: 'POST',
-    headers: {'Content-Type': 'application/json'},
-    body: JSON.stringify({id: id})
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ id: id })
   })
-    .then(function(r) { return r.json(); })
-    .then(function(d) {
+    .then(function (r) { return r.json(); })
+    .then(function (d) {
       if (d.ok) { loadHistory(promptRole); promptStatus.textContent = '✓ удалено'; }
       else { promptStatus.textContent = '✕ ' + (d.error || 'ошибка'); }
     });
 }
 
-document.getElementById('promptSave').addEventListener('click', function() {
+document.getElementById('promptSave').addEventListener('click', function () {
   var body = promptEditor.value.trim();
   if (!body) { promptStatus.textContent = '✕ тело промпта пустое'; return; }
-  var name = promptName.value.trim() || ('v' + new Date().toISOString().replace(/[:.]/g,'-').substring(0,16));
+  var name = promptName.value.trim() || ('v' + new Date().toISOString().replace(/[:.]/g, '-').substring(0, 16));
   fetch(VM_API + '/api/prompts/save', {
     method: 'POST',
-    headers: {'Content-Type': 'application/json'},
+    headers: { 'Content-Type': 'application/json' },
     body: JSON.stringify({
       role: promptRole, name: name, body: body,
       notes: promptNotes.value.trim(), is_active: true
     })
   })
-    .then(function(r) { return r.json(); })
-    .then(function(d) {
+    .then(function (r) { return r.json(); })
+    .then(function (d) {
       if (d.ok) {
         promptStatus.textContent = '✓ сохранено (новая версия)';
         promptName.value = ''; promptNotes.value = '';
@@ -615,10 +651,10 @@ document.getElementById('promptSave').addEventListener('click', function() {
     });
 });
 
-document.getElementById('promptTabExtract').addEventListener('click', function() { openPromptModal('extract'); });
-document.getElementById('promptTabDiff').addEventListener('click', function() { openPromptModal('diff'); });
-document.getElementById('promptTabClassify').addEventListener('click', function() { openPromptModal('classify'); });
-document.getElementById('promptBtnExtract').addEventListener('click', function() { openPromptModal('extract'); });
-document.getElementById('promptBtnDiff').addEventListener('click', function() { openPromptModal('diff'); });
-document.getElementById('promptBtnClassify').addEventListener('click', function() { openPromptModal('classify'); });
+document.getElementById('promptTabExtract').addEventListener('click', function () { openPromptModal('extract'); });
+document.getElementById('promptTabDiff').addEventListener('click', function () { openPromptModal('diff'); });
+document.getElementById('promptTabClassify').addEventListener('click', function () { openPromptModal('classify'); });
+document.getElementById('promptBtnExtract').addEventListener('click', function () { openPromptModal('extract'); });
+document.getElementById('promptBtnDiff').addEventListener('click', function () { openPromptModal('diff'); });
+document.getElementById('promptBtnClassify').addEventListener('click', function () { openPromptModal('classify'); });
 lucide.createIcons();

site/static/compare.js

@@ -215,6 +215,11 @@ function startCompareSSE(url, container, statusEl, callbacks) {
       container.innerHTML += '<div style="font-size:12px;color:var(--destructive);">✗ ' + escHtml(d.filename) + ': ' + escHtml(d.error) + '</div>';
     }
 
+    // ── done: завершение ──
+    else if (d.type === 'timeline') {
+      if (callbacks.onTimeline) callbacks.onTimeline(d);
+    }
+
     // ── done: завершение ──
     else if (d.type === 'done') {
       clearInterval(timer);

site/static/groups.js

@@ -27,6 +27,16 @@
  * Вызывается из runClassify() после успешной классификации.
  */
 async function loadGroupsAction() {
+  if (state.v3Mode) {
+    var v3Response = await fetch('/api/dossiers?batch_id=' + encodeURIComponent(state.batchId));
+    var v3Data = await v3Response.json();
+    if (!v3Data.ok) return;
+    state.dossiers = v3Data.dossiers || [];
+    renderDossiers(state);
+    stepDone('stepGroups');
+    return;
+  }
+
   var resp = await fetch(VM_API + '/api/groups?batch=' + state.batchId);
   var data = await resp.json();
   if (!data.ok || !data.groups) return;
@@ -115,6 +125,166 @@ function renderGroups(state) {
   lucide.createIcons();
 }
 
+
+function renderDossiers(appState) {
+  var diffBody = document.getElementById('diffBody');
+  var diffCard = document.getElementById('diffCard');
+  diffCard.style.display = 'block';
+  var summary = appState.v3Summary || {};
+  var html = '<div style="font-weight:600;margin-bottom:8px;">Досье: ' + appState.dossiers.length +
+    ' | требуется связать: ' + (summary.ambiguous || 0) + '</div>';
+
+  appState.dossiers.forEach(function(dossier) {
+    var aliases = (dossier.alias_numbers || []).map(escHtml).join(', ');
+    var docs = (dossier.documents || []).map(function(doc) {
+      return '<div style="padding:3px 0;font-size:12px;">' +
+        '<span style="color:var(--muted);">' + escHtml(doc.role || 'document') + '</span> ' +
+        escHtml(doc.filename || '') +
+        ' <span style="color:' + (doc.math_status === 'VERIFIED' ? 'var(--green)' : 'var(--muted)') + ';">' +
+        (doc.math_status === 'VERIFIED' ? '✓ арифметика' : '⚠ ' + escHtml(doc.math_status || 'UNKNOWN')) + '</span>' +
+        (doc.doc_date ? ' <span style="color:var(--muted);">' + escHtml(doc.doc_date) + '</span>' : '') +
+        '</div>';
+    }).join('');
+    html += '<section style="border-top:1px solid var(--brand-gray);padding:10px 0;">' +
+      '<div style="font-weight:600;">Договор №' + escHtml(dossier.canonical_number || '—') +
+      ' — ' + escHtml(dossier.counterparty_name || 'контрагент не определён') + '</div>' +
+      (aliases ? '<div style="font-size:12px;color:var(--muted);">Алиасы: ' + aliases + '</div>' : '') +
+      (!dossier.has_frame_contract ? '<div style="font-size:12px;color:var(--muted);">Рамочный договор не загружен</div>' : '') +
+      docs +
+      '<button class="btn btn-primary timeline-btn" type="button" data-dossier-id="' + escHtml(dossier.id) + '" style="margin-top:8px;">' +
+      '<i data-lucide="list-tree" style="width:16px;height:16px;"></i> Построить таймлайн</button>' +
+      '<div class="dossier-timeline" id="dossierTimeline_' + escHtml(dossier.id) + '" style="margin-top:8px;"></div>' +
+      '</section>';
+  });
+  if (summary.unlinked && summary.unlinked.length) {
+    html += '<div style="margin-top:8px;color:var(--destructive);">Не привязано документов: ' +
+      summary.unlinked.length + '</div>';
+  }
+  diffBody.innerHTML = html;
+  diffBody.querySelectorAll('.timeline-btn').forEach(function(button) {
+    button.addEventListener('click', function() {
+      loadDossierTimelineAction(this.getAttribute('data-dossier-id'), this);
+    });
+  });
+  lucide.createIcons();
+}
+
+
+function loadDossierTimelineAction(dossierId, button) {
+  var container = document.getElementById('dossierTimeline_' + dossierId);
+  var oldStatus = container.previousElementSibling;
+  if (oldStatus && oldStatus.classList.contains('dossier-status')) oldStatus.remove();
+  var status = document.createElement('div');
+  status.className = 'dossier-status';
+  status.style.cssText = 'font-size:12px;color:var(--muted);margin:6px 0;';
+  status.textContent = 'Построение таймлайна...';
+  container.parentNode.insertBefore(status, container);
+  container.innerHTML = '';
+  button.disabled = true;
+
+  var stream = startCompareSSE(
+    '/process-v2?dossier_id=' + encodeURIComponent(dossierId),
+    container,
+    status,
+    {
+      onTimeline: function(data) {
+        state.dossierTimelines[dossierId] = data;
+        renderDossierTimeline(dossierId, container, data);
+      },
+      onDone: function() {
+        button.disabled = false;
+        button.innerHTML = '<i data-lucide="list-tree" style="width:16px;height:16px;"></i> Обновить таймлайн';
+        status.textContent = 'Таймлайн обновлён';
+        lucide.createIcons();
+      },
+      onError: function(data) {
+        button.disabled = false;
+        status.textContent = 'Ошибка построения';
+        container.innerHTML = '<div style="color:var(--destructive);">' + escHtml(data.message) + '</div>';
+      },
+      onConnectionError: function() {
+        button.disabled = false;
+        status.textContent = 'Ошибка соединения';
+        container.innerHTML = '<div style="color:var(--destructive);">Ошибка соединения</div>';
+      }
+    },
+  );
+  state._activeCompare.es = stream.es;
+  state._activeCompare.timer = stream.timer;
+}
+
+
+function renderDossierTimeline(dossierId, container, data, asOf) {
+  asOf = asOf || new Date().toISOString().slice(0, 10);
+  var active = (data.timeline || []).filter(function(interval) {
+    return interval.valid_from && interval.valid_from <= asOf &&
+      (!interval.valid_to || interval.valid_to > asOf);
+  });
+  var byService = {};
+  (data.timeline || []).forEach(function(interval) {
+    var key = interval.service_item_id || interval.id;
+    (byService[key] || (byService[key] = [])).push(interval);
+  });
+  var html = '<label style="display:flex;align-items:center;gap:8px;font-size:12px;margin:8px 0;">' +
+    'Действующие услуги на дату <input class="timeline-date" type="date" value="' + escHtml(asOf) + '" style="height:30px;border:1px solid var(--brand-gray);border-radius:4px;padding:2px 6px;"></label>' +
+    '<div style="font-size:12px;color:var(--muted);margin-bottom:8px;">Активно: ' + active.length + ' интервалов</div>';
+  Object.keys(byService).forEach(function(serviceId) {
+    var intervals = byService[serviceId].slice().sort(function(left, right) {
+      return String(left.valid_from || '').localeCompare(String(right.valid_from || ''));
+    });
+    html += '<section style="border-top:1px solid var(--brand-gray);padding:8px 0;">' +
+      '<div style="font-weight:600;font-size:13px;">' + escHtml(intervals[0].raw_name || 'Услуга') + '</div>';
+    intervals.forEach(function(interval) {
+      var isActive = interval.valid_from && interval.valid_from <= asOf &&
+        (!interval.valid_to || interval.valid_to > asOf);
+      html += '<div style="font-size:12px;padding:3px 0;' + (isActive ? 'color:var(--green);font-weight:600;' : '') + '">[' + escHtml(interval.valid_from || '—') + ' — ' +
+        escHtml(interval.valid_to || '∞') + ') · ' + escHtml(interval.charge_type || 'unknown') +
+        ' · цена ' + escHtml(interval.price || '—') + ' · кол-во ' + escHtml(interval.qty || '—') +
+        ' · сумма ' + escHtml(interval.sum || '—') + (isActive ? ' · действует' : '') + '</div>';
+    });
+    html += '</section>';
+  });
+  if (data.exceptions && data.exceptions.length) {
+    html += '<section style="border-top:1px solid var(--brand-gray);padding:8px 0;">' +
+      '<div style="font-weight:600;color:var(--destructive);">Очередь исключений (' + data.exceptions.length + ')</div>';
+    data.exceptions.forEach(function(issue) {
+      html += '<div style="padding:6px 0;border-bottom:1px solid var(--brand-gray);">' +
+        '<div>' + escHtml(issue.title || issue.stage) + '</div>' +
+        (issue.source_quote ? '<div style="font-size:12px;color:var(--muted);">' + escHtml(issue.source_quote) + '</div>' : '');
+      (issue.options || []).forEach(function(option, index) {
+        var label = option.canonical_name || option.service_family || ('Вариант ' + (index + 1));
+        if (option.instance_anchor) label += ' · ' + option.instance_anchor;
+        html += '<button class="btn exception-choice" type="button" data-exception-id="' + escHtml(issue.id) +
+          '" data-option-index="' + index + '" style="margin:4px 4px 0 0;font-size:12px;min-height:32px;">' +
+          escHtml(label) + '</button>';
+      });
+      html += '</div>';
+    });
+    html += '</section>';
+  }
+  container.innerHTML = html;
+  container.querySelector('.timeline-date').addEventListener('change', function() {
+    renderDossierTimeline(dossierId, container, data, this.value);
+  });
+  container.querySelectorAll('.exception-choice').forEach(function(choice) {
+    choice.addEventListener('click', async function() {
+      var response = await fetch('/api/exceptions/' + encodeURIComponent(this.dataset.exceptionId) + '/resolve', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ resolution: { option_index: Number(this.dataset.optionIndex) } })
+      });
+      var result = await response.json();
+      if (!result.ok) {
+        this.disabled = false;
+        this.title = result.error || 'Не удалось сохранить решение';
+        return;
+      }
+      var button = document.querySelector('.timeline-btn[data-dossier-id="' + dossierId + '"]');
+      loadDossierTimelineAction(dossierId, button);
+    });
+  });
+}
+
 /**
  * renderUnresolvedCard(group) — Чистая HTML-функция: карточка нераспознанных файлов.
  * 

site/static/state.js

@@ -54,6 +54,10 @@ var state = {
 
   // ── Группы (бывший window._groupsData) ────────────────────
   groups: null,
+  v3Mode: false,
+  v3Summary: null,
+  dossiers: null,
+  dossierTimelines: {},
 
   // ── Активное сравнение ────────────────────────────────────
   _activeCompare: {