Improve LLM and CSV processing
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
+2
-16
@@ -8,13 +8,13 @@
|
||||
|
||||
import io
|
||||
import csv
|
||||
import os
|
||||
import zipfile
|
||||
import re
|
||||
import time
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
from .config import ENTITY_PATTERNS, COMPANY_PATTERN
|
||||
from .names import resolve_name
|
||||
|
||||
|
||||
def build_zip(files: List[Tuple[str, bytes]], mapping_csv: str = "") -> bytes:
|
||||
@@ -39,24 +39,10 @@ def build_zip(files: List[Tuple[str, bytes]], mapping_csv: str = "") -> bytes:
|
||||
# Дедупликация имён: имя+одинаковый контент → пропуск; имя+разный контент → суффикс
|
||||
seen: Dict[str, bytes] = {}
|
||||
|
||||
def _resolve_name(name: str, content: bytes) -> str:
|
||||
if name not in seen:
|
||||
seen[name] = content
|
||||
return name
|
||||
if seen[name] == content:
|
||||
return None # точный дубль — пропускаем
|
||||
base, ext = os.path.splitext(name)
|
||||
n = 2
|
||||
while f"{base}_{n}{ext}" in seen:
|
||||
n += 1
|
||||
resolved = f"{base}_{n}{ext}"
|
||||
seen[resolved] = content
|
||||
return resolved
|
||||
|
||||
with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
# Добавляем обфусцированные файлы
|
||||
for fname, content in files:
|
||||
out_name = _resolve_name(fname, content)
|
||||
out_name = resolve_name(fname, content, seen)
|
||||
if out_name is None:
|
||||
continue
|
||||
info = zipfile.ZipInfo(out_name)
|
||||
|
||||
Reference in New Issue
Block a user