feat: parser.py — слой парсера docx/pdf/doc/zip (без фильтрации)

site/parser.py

@@ -0,0 +1,193 @@
+"""
+Парсер документов — извлекает ВСЕ элементы без фильтрации и потерь.
+
+Поддерживает: docx, pdf, .doc (через libreoffice), zip.
+Возвращает полный слепок документа: абзацы (со стилями) + таблицы (все строки).
+"""
+
+import io
+import zipfile
+import subprocess
+import tempfile
+import os
+
+
+def parse(file_bytes: bytes, mime_type: str) -> dict:
+    """
+    Принимает сырые байты файла и MIME-тип.
+    Возвращает {"elements": [...], "errors": [...]}.
+    elements — все абзацы и таблицы в порядке документа.
+    """
+    result = {"elements": [], "errors": []}
+
+    if mime_type == "application/zip":
+        return _parse_zip(file_bytes)
+
+    if mime_type in ("application/vnd.openxmlformats-officedocument.wordprocessingml.document",):
+        return _parse_docx(file_bytes)
+
+    if mime_type in ("application/msword",):
+        return _parse_doc(file_bytes)
+
+    if mime_type == "application/pdf":
+        return _parse_pdf(file_bytes)
+
+    result["errors"].append(f"unsupported mime_type: {mime_type}")
+    return result
+
+
+# ── docx ──
+
+def _parse_docx(file_bytes: bytes) -> dict:
+    import docx
+
+    result = {"elements": [], "errors": []}
+    try:
+        doc = docx.Document(io.BytesIO(file_bytes))
+
+        # Собираем все параграфы и таблицы в порядке документа
+        body = doc.element.body
+        para_idx = 0
+        table_idx = 0
+
+        for child in body:
+            tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag
+
+            if tag == "p":
+                # Параграф
+                para = doc.paragraphs[para_idx]
+                para_idx += 1
+                if para.text.strip():
+                    result["elements"].append({
+                        "type": "paragraph",
+                        "style": para.style.name if para.style else "",
+                        "text": para.text,
+                    })
+
+            elif tag == "tbl":
+                # Таблица
+                table = doc.tables[table_idx]
+                table_idx += 1
+                rows = []
+                for row in table.rows:
+                    cells = [cell.text for cell in row.cells]
+                    rows.append(cells)
+                result["elements"].append({
+                    "type": "table",
+                    "rows": rows,
+                })
+
+    except Exception as e:
+        result["errors"].append(f"docx parse error: {e}")
+
+    return result
+
+
+# ── .doc (через libreoffice) ──
+
+def _parse_doc(file_bytes: bytes) -> dict:
+    result = {"elements": [], "errors": []}
+    try:
+        with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as tmp_doc:
+            tmp_doc.write(file_bytes)
+            tmp_doc_path = tmp_doc.name
+
+        tmp_dir = tempfile.mkdtemp()
+        subprocess.run(
+            ["libreoffice", "--headless", "--convert-to", "docx", "--outdir", tmp_dir, tmp_doc_path],
+            timeout=30, capture_output=True,
+        )
+
+        # Находим получившийся docx
+        docx_files = [f for f in os.listdir(tmp_dir) if f.endswith(".docx")]
+        if docx_files:
+            with open(os.path.join(tmp_dir, docx_files[0]), "rb") as f:
+                result = _parse_docx(f.read())
+        else:
+            result["errors"].append("libreoffice conversion produced no docx")
+
+        os.unlink(tmp_doc_path)
+        for f in os.listdir(tmp_dir):
+            os.unlink(os.path.join(tmp_dir, f))
+        os.rmdir(tmp_dir)
+
+    except FileNotFoundError:
+        result["errors"].append("libreoffice not installed")
+    except Exception as e:
+        result["errors"].append(f"doc parse error: {e}")
+
+    return result
+
+
+# ── PDF ──
+
+def _parse_pdf(file_bytes: bytes) -> dict:
+    result = {"elements": [], "errors": []}
+    try:
+        import pdfplumber
+
+        with pdfplumber.open(io.BytesIO(file_bytes)) as pdf:
+            for page in pdf.pages:
+                # Текст страницы
+                text = page.extract_text()
+                if text:
+                    for line in text.split("\n"):
+                        line = line.strip()
+                        if line:
+                            result["elements"].append({
+                                "type": "paragraph",
+                                "style": "",
+                                "text": line,
+                            })
+
+                # Таблицы на странице
+                tables = page.extract_tables()
+                for tbl in tables:
+                    if tbl:
+                        result["elements"].append({
+                            "type": "table",
+                            "rows": tbl,
+                        })
+
+    except Exception as e:
+        result["errors"].append(f"pdf parse error: {e}")
+
+    return result
+
+
+# ── ZIP ──
+
+def _parse_zip(file_bytes: bytes) -> dict:
+    result = {"elements": [], "errors": [], "files": []}
+    try:
+        with zipfile.ZipFile(io.BytesIO(file_bytes)) as zf:
+            for name in zf.namelist():
+                if name.endswith("/"):
+                    continue
+                file_data = zf.read(name)
+                ext = name.lower().split(".")[-1] if "." in name else ""
+
+                mime_map = {
+                    "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                    "doc": "application/msword",
+                    "pdf": "application/pdf",
+                }
+                mime = mime_map.get(ext, "")
+
+                if mime:
+                    parsed = parse(file_data, mime)
+                    result["files"].append({
+                        "filename": name,
+                        "elements": parsed["elements"],
+                        "errors": parsed["errors"],
+                    })
+                else:
+                    result["files"].append({
+                        "filename": name,
+                        "elements": [],
+                        "errors": [f"unsupported extension: {ext}"],
+                    })
+    except Exception as e:
+        result["errors"].append(f"zip parse error: {e}")
+
+    return result