contracts-app/site/parser.py

"""
Парсер документов — извлекает ВСЕ элементы без фильтрации и потерь.

Поддерживает: docx, pdf, .doc (через libreoffice), zip.
Возвращает полный слепок документа: абзацы (со стилями) + таблицы (все строки).
"""

import io
import zipfile
import subprocess
import tempfile
import os


def parse(file_bytes: bytes, mime_type: str) -> dict:
    """
    Принимает сырые байты файла и MIME-тип.
    Возвращает {"elements": [...], "errors": [...]}.
    elements — все абзацы и таблицы в порядке документа.
    """
    result = {"elements": [], "errors": []}

    if mime_type == "application/zip":
        return _parse_zip(file_bytes)

    if mime_type in ("application/vnd.openxmlformats-officedocument.wordprocessingml.document",):
        return _parse_docx(file_bytes)

    if mime_type in ("application/msword",):
        return _parse_doc(file_bytes)

    if mime_type == "application/pdf":
        return _parse_pdf(file_bytes)

    result["errors"].append(f"unsupported mime_type: {mime_type}")
    return result


# ── docx ──

def _parse_docx(file_bytes: bytes) -> dict:
    import docx

    result = {"elements": [], "errors": []}
    try:
        doc = docx.Document(io.BytesIO(file_bytes))

        # Собираем все параграфы и таблицы в порядке документа
        body = doc.element.body
        para_idx = 0
        table_idx = 0

        for child in body:
            tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag

            if tag == "p":
                # Параграф
                para = doc.paragraphs[para_idx]
                para_idx += 1
                if para.text.strip():
                    result["elements"].append({
                        "type": "paragraph",
                        "style": para.style.name if para.style else "",
                        "text": para.text,
                    })

            elif tag == "tbl":
                # Таблица
                table = doc.tables[table_idx]
                table_idx += 1
                rows = []
                for row in table.rows:
                    cells = [cell.text for cell in row.cells]
                    rows.append(cells)
                result["elements"].append({
                    "type": "table",
                    "rows": rows,
                })

    except Exception as e:
        result["errors"].append(f"docx parse error: {e}")

    return result


# ── .doc (через libreoffice) ──

def _parse_doc(file_bytes: bytes) -> dict:
    result = {"elements": [], "errors": []}
    try:
        with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as tmp_doc:
            tmp_doc.write(file_bytes)
            tmp_doc_path = tmp_doc.name

        tmp_dir = tempfile.mkdtemp()
        subprocess.run(
            ["libreoffice", "--headless", "--convert-to", "docx", "--outdir", tmp_dir, tmp_doc_path],
            timeout=30, capture_output=True,
        )

        # Находим получившийся docx
        docx_files = [f for f in os.listdir(tmp_dir) if f.endswith(".docx")]
        if docx_files:
            with open(os.path.join(tmp_dir, docx_files[0]), "rb") as f:
                result = _parse_docx(f.read())
        else:
            result["errors"].append("libreoffice conversion produced no docx")

        os.unlink(tmp_doc_path)
        for f in os.listdir(tmp_dir):
            os.unlink(os.path.join(tmp_dir, f))
        os.rmdir(tmp_dir)

    except FileNotFoundError:
        result["errors"].append("libreoffice not installed")
    except Exception as e:
        result["errors"].append(f"doc parse error: {e}")

    return result


# ── PDF ──

def _parse_pdf(file_bytes: bytes) -> dict:
    result = {"elements": [], "errors": []}

    # Попробовать camelot (pdfium, без системных зависимостей)
    tables_from_camelot = False
    try:
        import camelot
        for flavor in ("lattice", "stream"):
            try:
                ctables = camelot.read_pdf(io.BytesIO(file_bytes), pages="all", flavor=flavor)
                for ct in ctables:
                    rows = [list(ct.df.columns)] + ct.df.values.tolist()
                    result["elements"].append({
                        "type": "table",
                        "rows": rows,
                        "page": ct.page,
                    })
                    tables_from_camelot = True
            except Exception:
                pass
    except ImportError:
        result["errors"].append("camelot not installed, using pdfplumber")
    except Exception as e:
        result["errors"].append(f"camelot error: {e}")

    # pdfplumber: текст + таблицы (если tabula не сработал)
    try:
        import pdfplumber
        with pdfplumber.open(io.BytesIO(file_bytes)) as pdf:
            for page in pdf.pages:
                pn = page.page_number
                text = page.extract_text()
                if text:
                    for line in text.split("\n"):
                        line = line.strip()
                        if line:
                            result["elements"].append({
                                "type": "paragraph",
                                "style": "",
                                "text": line,
                                "page": pn,
                            })
                if not tables_from_camelot:
                    tables = page.extract_tables()
                    for tbl in tables:
                        if tbl:
                            result["elements"].append({
                                "type": "table",
                                "rows": tbl,
                                "page": pn,
                            })
    except Exception as e:
        result["errors"].append(f"pdfplumber: {e}")

    return result


# ── ZIP ──

def _parse_zip(file_bytes: bytes) -> dict:
    result = {"elements": [], "errors": [], "files": []}
    try:
        with zipfile.ZipFile(io.BytesIO(file_bytes)) as zf:
            for name in zf.namelist():
                if name.endswith("/"):
                    continue
                file_data = zf.read(name)
                ext = name.lower().split(".")[-1] if "." in name else ""

                mime_map = {
                    "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
                    "doc": "application/msword",
                    "pdf": "application/pdf",
                }
                mime = mime_map.get(ext, "")

                if mime:
                    parsed = parse(file_data, mime)
                    result["files"].append({
                        "filename": name,
                        "elements": parsed["elements"],
                        "errors": parsed["errors"],
                    })
                else:
                    result["files"].append({
                        "filename": name,
                        "elements": [],
                        "errors": [f"unsupported extension: {ext}"],
                    })
    except Exception as e:
        result["errors"].append(f"zip parse error: {e}")

    return result