Improve LLM and CSV processing

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Repinoid
2026-10-23 10:09:34 +03:00
co-authored by Copilot
parent aa1d15509b
commit 49171c93f9
14 changed files with 315 additions and 92 deletions
+2 -16
View File
@@ -8,13 +8,13 @@
import io
import csv
import os
import zipfile
import re
import time
from typing import Dict, List, Tuple
from .config import ENTITY_PATTERNS, COMPANY_PATTERN
from .names import resolve_name
def build_zip(files: List[Tuple[str, bytes]], mapping_csv: str = "") -> bytes:
@@ -39,24 +39,10 @@ def build_zip(files: List[Tuple[str, bytes]], mapping_csv: str = "") -> bytes:
# Дедупликация имён: имя+одинаковый контент → пропуск; имя+разный контент → суффикс
seen: Dict[str, bytes] = {}
def _resolve_name(name: str, content: bytes) -> str:
if name not in seen:
seen[name] = content
return name
if seen[name] == content:
return None # точный дубль — пропускаем
base, ext = os.path.splitext(name)
n = 2
while f"{base}_{n}{ext}" in seen:
n += 1
resolved = f"{base}_{n}{ext}"
seen[resolved] = content
return resolved
with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf:
# Добавляем обфусцированные файлы
for fname, content in files:
out_name = _resolve_name(fname, content)
out_name = resolve_name(fname, content, seen)
if out_name is None:
continue
info = zipfile.ZipInfo(out_name)