processing: clarify per-file stages and outcomes

Expose extraction, analysis, and replacement progress in the file table; preserve row mapping for duplicates and collapsed archives.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Repinoid
2026-10-24 11:54:09 +03:00
co-authored by Copilot
parent f476d6cd14
commit 1b0bcd298a
8 changed files with 359 additions and 138 deletions
+13 -4
View File
@@ -14,7 +14,7 @@ import json
import random
import time
import logging
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Dict, List, Optional
from .config import (
@@ -294,13 +294,22 @@ def scan_llm_ner(all_texts: Dict[str, str], mapping: Dict[str, str],
with ThreadPoolExecutor(max_workers=_LLM_CONCURRENCY) as _ex:
futures = [_ex.submit(_call_llm, _build_llm_prompt(c, mapping.keys()), llm_client)
for c in chunks]
all_ent = []
for fut in futures:
chunk_results = [None] * len(futures)
future_indexes = {future: index for index, future in enumerate(futures)}
completed = 0
for fut in as_completed(futures):
index = future_indexes[fut]
try:
all_ent.extend(fut.result())
chunk_results[index] = fut.result()
except Exception as e:
llm_failures += 1
log.warning("LLM NER chunk failed in %r: %s", fname, e)
chunk_results[index] = []
completed += 1
if file_progress:
file_progress("file_chunk", fname,
chunks_done=completed, chunks_total=len(chunks))
all_ent = [ent for chunk_result in chunk_results for ent in chunk_result]
else:
all_ent = []
for k, c in enumerate(chunks):