processing: clarify per-file stages and outcomes
Expose extraction, analysis, and replacement progress in the file table; preserve row mapping for duplicates and collapsed archives. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
+13
-4
@@ -14,7 +14,7 @@ import json
|
||||
import random
|
||||
import time
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from .config import (
|
||||
@@ -294,13 +294,22 @@ def scan_llm_ner(all_texts: Dict[str, str], mapping: Dict[str, str],
|
||||
with ThreadPoolExecutor(max_workers=_LLM_CONCURRENCY) as _ex:
|
||||
futures = [_ex.submit(_call_llm, _build_llm_prompt(c, mapping.keys()), llm_client)
|
||||
for c in chunks]
|
||||
all_ent = []
|
||||
for fut in futures:
|
||||
chunk_results = [None] * len(futures)
|
||||
future_indexes = {future: index for index, future in enumerate(futures)}
|
||||
completed = 0
|
||||
for fut in as_completed(futures):
|
||||
index = future_indexes[fut]
|
||||
try:
|
||||
all_ent.extend(fut.result())
|
||||
chunk_results[index] = fut.result()
|
||||
except Exception as e:
|
||||
llm_failures += 1
|
||||
log.warning("LLM NER chunk failed in %r: %s", fname, e)
|
||||
chunk_results[index] = []
|
||||
completed += 1
|
||||
if file_progress:
|
||||
file_progress("file_chunk", fname,
|
||||
chunks_done=completed, chunks_total=len(chunks))
|
||||
all_ent = [ent for chunk_result in chunk_results for ent in chunk_result]
|
||||
else:
|
||||
all_ent = []
|
||||
for k, c in enumerate(chunks):
|
||||
|
||||
Reference in New Issue
Block a user