Nail/contracts-app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ZIP: распаковка и парсинг каждого файла внутри, v1.3

Browse Source
This commit is contained in:
naeel
2026-06-16 14:56:12 +04:00
parent 1f7c392934
commit 21f8c0b072
2 changed files with 73 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
site/app.py
+65 -17
View File
@@ -165,25 +165,73 @@ class ContractsApp:
raw = doc["original_bytes"]
file_data = bytes(raw) if isinstance(raw, memoryview) else raw
# Парсинг
pr = parser_mod.parse(file_data, s["mime_type"])
elements = []
if s["mime_type"] == "application/zip" and "files" in pr:
for zf in pr["files"]:
elements.extend(zf.get("elements", []))
# Парсинг: ZIP — распаковать и парсить каждый файл отдельно
if s["mime_type"] == "application/zip":
import io as io_mod, zipfile as zf_mod
zip_names = []
try:
with zf_mod.ZipFile(io_mod.BytesIO(file_data)) as zf:
for zname in zf.namelist():
if zname.endswith("/"):
continue
zip_names.append(zname)
zdata = zf.read(zname)
zmime = mimeutil.guess_mime(zname) or "application/octet-stream"
zbasename = zname.split("/")[-1] if "/" in zname else zname
# Сохранить как отдельный документ
db.execute(
"INSERT INTO documents (filename, mime_type, original_bytes, status) VALUES (%s,%s,%s,'uploaded')",
(zbasename, zmime, zdata),
)
zdoc, _ = db.query_one(
"SELECT id FROM documents WHERE filename=%s ORDER BY created_at DESC LIMIT 1",
(zbasename,),
)
zdoc_id = zdoc["id"] if zdoc else None
if zdoc_id:
db.execute(
"INSERT INTO supplements (contract_id, document_id, type) VALUES (%s,%s,'initial')",
(cid, str(zdoc_id)),
)
# Парсинг внутреннего файла
zf_start = time_mod.time()
total_bytes += len(zdata)
yield f"data: {json.dumps({'type': 'file_start', 'name': zbasename, 'bytes': len(zdata)})}\n\n"
try:
zpr = parser_mod.parse(zdata, zmime)
zelements = zpr.get("elements", [])
ztext = textify.to_text(zelements) if zelements else ""
db.execute(
"UPDATE documents SET parsed_text=%s, status='parsed' WHERE id=%s",
(ztext, str(zdoc_id)),
)
zelapsed = round(time_mod.time() - zf_start, 1)
files_processed += 1
yield f"data: {json.dumps({'type': 'file_done', 'name': zbasename, 'time_s': zelapsed, 'elements': len(zelements)})}\n\n"
except Exception as e:
yield f"data: {json.dumps({'type': 'file_error', 'name': zbasename, 'error': str(e)})}\n\n"
except Exception as e:
yield f"data: {json.dumps({'type': 'file_error', 'name': s['filename'], 'error': 'ZIP: ' + str(e)})}\n\n"
# ZIP сам — expanded
db.execute("UPDATE documents SET status='expanded' WHERE id=%s", (s["doc_id"],))
yield f"data: {json.dumps({'type': 'zip_expanded', 'name': s['filename'], 'count': len(zip_names)})}\n\n"
else:
pr = parser_mod.parse(file_data, s["mime_type"])
elements = pr.get("elements", [])
text = textify.to_text(elements) if elements else ""
db.execute(
"UPDATE documents SET parsed_text=%s, status='parsed' WHERE id=%s",
(text, str(s["doc_id"])),
)
elapsed = round(time_mod.time() - file_start, 1)
files_processed += 1
yield f"data: {json.dumps({'type': 'file_done', 'name': s['filename'], 'time_s': elapsed, 'elements': len(elements)})}\n\n"
text = textify.to_text(elements) if elements else ""
db.execute(
"UPDATE documents SET parsed_text=%s, status='parsed' WHERE id=%s",
(text, str(s["doc_id"])),
)
elapsed = round(time_mod.time() - file_start, 1)
files_processed += 1
yield f"data: {json.dumps({'type': 'file_done', 'name': s['filename'], 'time_s': elapsed, 'elements': len(elements)})}\n\n"
total_time = round(time_mod.time() - start_time, 1)
yield f"data: {json.dumps({'type': 'summary', 'total_time_s': total_time, 'total_bytes': total_bytes, 'files_processed': files_processed})}\n\n"