diff --git a/deploy/convert_server.py b/deploy/convert_server.py index add6a77..c27853c 100755 --- a/deploy/convert_server.py +++ b/deploy/convert_server.py @@ -14,6 +14,7 @@ db_prompts.seed_defaults() execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_raw text") execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS classify_input text") execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS zip_source text") +execute("ALTER TABLE documents ADD COLUMN IF NOT EXISTS content_hash text") # ── DB modules ──────────────────────────────────────────────────────────── from db import supplements as db_supplements diff --git a/deploy/db/documents.py b/deploy/db/documents.py index 76f9e64..d711689 100644 --- a/deploy/db/documents.py +++ b/deploy/db/documents.py @@ -2,12 +2,12 @@ from .connection import query, execute, execute_returning -def insert(filename, mime_type, original_bytes, status="uploaded", batch_id=None, zip_source=None): +def insert(filename, mime_type, original_bytes, status="uploaded", batch_id=None, zip_source=None, content_hash=None): """Insert document, return row dict.""" return execute_returning( - """INSERT INTO documents (filename, mime_type, original_bytes, status, batch_id, zip_source) - VALUES (%s, %s, %s, %s, %s, %s) RETURNING *""", - (filename, mime_type, original_bytes, status, batch_id, zip_source), + """INSERT INTO documents (filename, mime_type, original_bytes, status, batch_id, zip_source, content_hash) + VALUES (%s, %s, %s, %s, %s, %s, %s) RETURNING *""", + (filename, mime_type, original_bytes, status, batch_id, zip_source, content_hash), ) @@ -16,6 +16,15 @@ def get(doc_id): return rows[0] if rows else None +def get_by_hash(batch_id, content_hash): + """Find document by content hash within batch.""" + rows = query( + "SELECT id FROM documents WHERE batch_id = %s AND content_hash = %s LIMIT 1", + (batch_id, content_hash), + ) + return rows[0] if rows else None + + def set_parsed(doc_id, elements_json): """Update elements_json + status='parsed'.""" import json diff --git a/deploy/services/upload.py b/deploy/services/upload.py index 6737d73..960a5fc 100644 --- a/deploy/services/upload.py +++ b/deploy/services/upload.py @@ -77,6 +77,15 @@ def store_document(filename: str, file_data: bytes, contract_id: str = "", if not filename or not file_data: return {"ok": False, "error": "no file"} + import hashlib + content_hash = hashlib.sha256(file_data).hexdigest() + + # Dedup: same content in same batch → skip + if batch_id: + existing = documents.get_by_hash(batch_id, content_hash) + if existing: + return {"ok": True, "duplicate_of": existing["id"], "filename": filename} + mime = _mime_for(filename) b64 = base64.b64encode(file_data).decode() @@ -86,7 +95,8 @@ def store_document(filename: str, file_data: bytes, contract_id: str = "", except Exception: pass - doc = documents.insert(filename, mime, b64, batch_id=batch_id, zip_source=zip_source) + doc = documents.insert(filename, mime, b64, batch_id=batch_id, zip_source=zip_source, + content_hash=content_hash) if not contract_id: from datetime import datetime