feat: content_hash + дедупликация при загрузке

deploy/convert_server.py

@@ -14,6 +14,7 @@ db_prompts.seed_defaults()
 execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_raw text")
 execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_input text")
 execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS zip_source text")
+execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS content_hash text")
 
 # ── DB modules ────────────────────────────────────────────────────────────
 from db import supplements as db_supplements

deploy/db/documents.py

@@ -2,12 +2,12 @@
 from .connection import query, execute, execute_returning
 
 
-def insert(filename, mime_type, original_bytes, status="uploaded", batch_id=None, zip_source=None):
+def insert(filename, mime_type, original_bytes, status="uploaded", batch_id=None, zip_source=None, content_hash=None):
     """Insert document, return row dict."""
     return execute_returning(
-        """INSERT INTO documents (filename, mime_type, original_bytes, status, batch_id, zip_source)
-           VALUES (%s, %s, %s, %s, %s, %s) RETURNING *""",
-        (filename, mime_type, original_bytes, status, batch_id, zip_source),
+        """INSERT INTO documents (filename, mime_type, original_bytes, status, batch_id, zip_source, content_hash)
+           VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING *""",
+        (filename, mime_type, original_bytes, status, batch_id, zip_source, content_hash),
     )
 
 
@@ -16,6 +16,15 @@ def get(doc_id):
     return rows[0] if rows else None
 
 
+def get_by_hash(batch_id, content_hash):
+    """Find document by content hash within batch."""
+    rows = query(
+        "SELECT id FROM documents WHERE batch_id = %s AND content_hash = %s LIMIT 1",
+        (batch_id, content_hash),
+    )
+    return rows[0] if rows else None
+
+
 def set_parsed(doc_id, elements_json):
     """Update elements_json + status='parsed'."""
     import json

deploy/services/upload.py

@@ -77,6 +77,15 @@ def store_document(filename: str, file_data: bytes, contract_id: str = "",
     if not filename or not file_data:
         return {"ok": False, "error": "no file"}
 
+    import hashlib
+    content_hash = hashlib.sha256(file_data).hexdigest()
+
+    # Dedup: same content in same batch → skip
+    if batch_id:
+        existing = documents.get_by_hash(batch_id, content_hash)
+        if existing:
+            return {"ok": True, "duplicate_of": existing["id"], "filename": filename}
+
     mime = _mime_for(filename)
     b64 = base64.b64encode(file_data).decode()
 
@@ -86,7 +95,8 @@ def store_document(filename: str, file_data: bytes, contract_id: str = "",
         except Exception:
             pass
 
-    doc = documents.insert(filename, mime, b64, batch_id=batch_id, zip_source=zip_source)
+    doc = documents.insert(filename, mime, b64, batch_id=batch_id, zip_source=zip_source,
+                           content_hash=content_hash)
 
     if not contract_id:
         from datetime import datetime