fix: paragraph-level docx replace, LLM-NER, НУБЕС whitelist, COMPANY_PATTERN limit — v0.6/v1.0.189
Deploy contracts-flask / validate (push) Successful in 0s
Deploy contracts-flask / validate (push) Successful in 0s
This commit is contained in:
+3
-1
@@ -133,7 +133,9 @@ def api_drhider():
|
||||
|
||||
def _process():
|
||||
try:
|
||||
zip_data, _csv = obfuscate_files(files)
|
||||
from services.llm_client import HttpxLLMClient
|
||||
llm = HttpxLLMClient(LLM_URL, LLM_KEY, LLM_MODEL) if LLM_KEY else None
|
||||
zip_data, _csv = obfuscate_files(files, llm_client=llm)
|
||||
with _drhider_lock:
|
||||
_drhider_jobs[job_id] = {"done": True, "data": zip_data, "error": None}
|
||||
except Exception as e:
|
||||
|
||||
+30
-10
@@ -6,7 +6,7 @@ DrHider — обфускация документов (двухпроходна
|
||||
|
||||
Согласованность: одна и та же сущность во всех файлах → одно и то же фиктивное значение.
|
||||
"""
|
||||
DRHIDER_VERSION = "0.5"
|
||||
DRHIDER_VERSION = "0.6"
|
||||
import io
|
||||
import csv
|
||||
import re
|
||||
@@ -40,7 +40,13 @@ ENTITY_PATTERNS: Dict[str, str] = {
|
||||
|
||||
# Фирмы — обнаружение по шаблону
|
||||
COMPANY_PATTERN = re.compile(
|
||||
r'(?:ООО|ЗАО|ОАО|АО|ПАО|ИП|ТОО)\s+(?:«[^»]+»|"[^"]+"|[А-ЯA-Z][\w\s\-\.]+)',
|
||||
r'(?:ООО|ЗАО|ОАО|АО|ПАО|ИП|ТОО)\s+(?:«[^»]+»|"[^"]+"|\u201c[^\u201d]+\u201d|[А-ЯA-Z][\w\-\.]{2,40})',
|
||||
re.IGNORECASE
|
||||
)
|
||||
|
||||
# ФИО — Фамилия + инициалы
|
||||
PERSON_PATTERN = re.compile(
|
||||
r'\b[А-ЯA-Z][а-яa-z]+\s+[А-ЯA-Z]\.[А-ЯA-Z]\.',
|
||||
re.IGNORECASE
|
||||
)
|
||||
|
||||
@@ -335,12 +341,20 @@ class TwoPassObfuscator:
|
||||
generator = ENTITY_GENERATORS.get(entity_type, lambda x: "XXX")
|
||||
self._mapping[original] = generator(original)
|
||||
|
||||
# Компании
|
||||
# Компании (кроме НУБЕС — это Исполнитель)
|
||||
for match in COMPANY_PATTERN.finditer(text):
|
||||
original = match.group(0).strip()
|
||||
if original and original not in self._mapping:
|
||||
if re.search(r'НУБЕС|NUBES', original, re.IGNORECASE):
|
||||
continue # Исполнитель — не заменяем
|
||||
self._mapping[original] = generate_company(original)
|
||||
|
||||
# ФИО (Фамилия И.О.)
|
||||
for match in PERSON_PATTERN.finditer(text):
|
||||
original = match.group(0).strip()
|
||||
if original and original not in self._mapping:
|
||||
self._mapping[original] = generate_person(original)
|
||||
|
||||
def _scan_llm_ner(self, all_texts: Dict[str, str]):
|
||||
"""LLM NER для обнаружения имён и адресов во всех файлах."""
|
||||
combined = "\n\n---FILE---\n\n".join(
|
||||
@@ -384,19 +398,25 @@ class TwoPassObfuscator:
|
||||
# --- Проход 2: замена ---
|
||||
|
||||
def _replace_in_docx(self, doc) -> bytes:
|
||||
"""Заменить сущности в docx-документе (in-place)."""
|
||||
# Замена в параграфах
|
||||
"""Заменить сущности в docx-документе. Склеиваем runs → заменяем → пишем в первый run, остальные очищаем."""
|
||||
for para in doc.paragraphs:
|
||||
for run in para.runs:
|
||||
run.text = self._apply_replacements(run.text)
|
||||
full_text = "".join(run.text for run in para.runs)
|
||||
replaced = self._apply_replacements(full_text)
|
||||
if replaced != full_text and para.runs:
|
||||
para.runs[0].text = replaced
|
||||
for run in para.runs[1:]:
|
||||
run.text = ""
|
||||
|
||||
# Замена в таблицах
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
for para in cell.paragraphs:
|
||||
for run in para.runs:
|
||||
run.text = self._apply_replacements(run.text)
|
||||
full_text = "".join(run.text for run in para.runs)
|
||||
replaced = self._apply_replacements(full_text)
|
||||
if replaced != full_text and para.runs:
|
||||
para.runs[0].text = replaced
|
||||
for run in para.runs[1:]:
|
||||
run.text = ""
|
||||
|
||||
buf = io.BytesIO()
|
||||
doc.save(buf)
|
||||
|
||||
@@ -64,7 +64,7 @@
|
||||
<a href="/">Сверка договоров</a>
|
||||
<span class="sep">|</span>
|
||||
<strong>DrHider</strong>
|
||||
<span style="font-size:11px;color:var(--muted);">v0.5</span>
|
||||
<span style="font-size:11px;color:var(--muted);">v0.6</span>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
|
||||
@@ -73,7 +73,7 @@
|
||||
<body>
|
||||
<div class="topbar">
|
||||
<img src="/static/logo.svg" alt="Nubes">
|
||||
<span class="title">Сверка договоров — LLM AI-driven Event Sourcing <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.0.188-flask</span></span>
|
||||
<span class="title">Сверка договоров — LLM AI-driven Event Sourcing <span style="font-weight:400;color:var(--muted);font-size:12px;">v1.0.189-flask</span></span>
|
||||
<div id="pipelineStepper" style="display:flex;gap:8px;font-size:11px;align-items:center;color:var(--muted);">
|
||||
<span id="stepUpload">○ Загрузка</span><span>→</span>
|
||||
<span id="stepClassify">○ Классификация</span><span>→</span>
|
||||
|
||||
Reference in New Issue
Block a user