feat: белый список форматов + сводка батча по типам
This commit is contained in:
@@ -103,10 +103,11 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
|||||||
garbage = 0
|
garbage = 0
|
||||||
json_fixes = 0
|
json_fixes = 0
|
||||||
json_total = 0
|
json_total = 0
|
||||||
|
type_counts = {} # doc_type → count for batch summary
|
||||||
|
|
||||||
def _classify_one(doc):
|
def _classify_one(doc):
|
||||||
"""Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
|
"""Классифицировать один документ: фильтр → выжимка → LLM → сохранить."""
|
||||||
nonlocal garbage, json_fixes, json_total
|
nonlocal garbage, json_fixes, json_total, type_counts
|
||||||
try:
|
try:
|
||||||
# Stage 1: garbage by filename (0 tokens)
|
# Stage 1: garbage by filename (0 tokens)
|
||||||
if _is_garbage_by_filename(doc["filename"]):
|
if _is_garbage_by_filename(doc["filename"]):
|
||||||
@@ -127,6 +128,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
|||||||
json_total += 1
|
json_total += 1
|
||||||
if needed_fix:
|
if needed_fix:
|
||||||
json_fixes += 1
|
json_fixes += 1
|
||||||
|
dtype = result.get("doc_type", "other")
|
||||||
|
type_counts[dtype] = type_counts.get(dtype, 0) + 1
|
||||||
_db.set_classification(
|
_db.set_classification(
|
||||||
doc["id"],
|
doc["id"],
|
||||||
result.get("doc_type", "other"),
|
result.get("doc_type", "other"),
|
||||||
@@ -151,7 +154,8 @@ def classify_batch(batch_id, llm_client=None, repo=None):
|
|||||||
failed += 1
|
failed += 1
|
||||||
|
|
||||||
return {"ok": True, "total": total, "done": done, "failed": failed,
|
return {"ok": True, "total": total, "done": done, "failed": failed,
|
||||||
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3)}
|
"garbage": garbage, "json_fix_rate": round(json_fixes / max(json_total, 1), 3),
|
||||||
|
"types": type_counts, "summary": f"{total} total, {done} classified, {failed} failed, {garbage} garbage"}
|
||||||
|
|
||||||
|
|
||||||
def _safe_json_parse(raw):
|
def _safe_json_parse(raw):
|
||||||
|
|||||||
@@ -4,6 +4,15 @@ from db import documents, contracts, supplements
|
|||||||
from .parse import parse_file
|
from .parse import parse_file
|
||||||
|
|
||||||
MAX_FILE_SIZE = 100 * 1024 * 1024 # 100 MB
|
MAX_FILE_SIZE = 100 * 1024 * 1024 # 100 MB
|
||||||
|
ALLOWED_EXTENSIONS = {"pdf", "docx", "doc", "zip"}
|
||||||
|
|
||||||
|
|
||||||
|
def _check_format(filename: str) -> str | None:
|
||||||
|
"""Validate file extension. Returns error string or None."""
|
||||||
|
ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
|
||||||
|
if ext not in ALLOWED_EXTENSIONS:
|
||||||
|
return f"unsupported format: .{ext} (allowed: {', '.join(sorted(ALLOWED_EXTENSIONS))})"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
# ── Чистые функции (тестируемы без HTTP) ──────────────────────
|
# ── Чистые функции (тестируемы без HTTP) ──────────────────────
|
||||||
@@ -77,6 +86,10 @@ def store_document(filename: str, file_data: bytes, contract_id: str = "",
|
|||||||
if not filename or not file_data:
|
if not filename or not file_data:
|
||||||
return {"ok": False, "error": "no file"}
|
return {"ok": False, "error": "no file"}
|
||||||
|
|
||||||
|
fmt_err = _check_format(filename)
|
||||||
|
if fmt_err:
|
||||||
|
return {"ok": False, "error": fmt_err}
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
content_hash = hashlib.sha256(file_data).hexdigest()
|
content_hash = hashlib.sha256(file_data).hexdigest()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user