diff --git a/requirements.txt b/requirements.txt index de46a10..dbe9f25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ python-docx requests psycopg2-binary python-dotenv +pdfplumber diff --git a/site/parser.py b/site/parser.py new file mode 100644 index 0000000..d0ba748 --- /dev/null +++ b/site/parser.py @@ -0,0 +1,193 @@ +""" +Парсер документов — извлекает ВСЕ элементы без фильтрации и потерь. + +Поддерживает: docx, pdf, .doc (через libreoffice), zip. +Возвращает полный слепок документа: абзацы (со стилями) + таблицы (все строки). +""" + +import io +import zipfile +import subprocess +import tempfile +import os + + +def parse(file_bytes: bytes, mime_type: str) -> dict: + """ + Принимает сырые байты файла и MIME-тип. + Возвращает {"elements": [...], "errors": [...]}. + elements — все абзацы и таблицы в порядке документа. + """ + result = {"elements": [], "errors": []} + + if mime_type == "application/zip": + return _parse_zip(file_bytes) + + if mime_type in ("application/vnd.openxmlformats-officedocument.wordprocessingml.document",): + return _parse_docx(file_bytes) + + if mime_type in ("application/msword",): + return _parse_doc(file_bytes) + + if mime_type == "application/pdf": + return _parse_pdf(file_bytes) + + result["errors"].append(f"unsupported mime_type: {mime_type}") + return result + + +# ── docx ── + +def _parse_docx(file_bytes: bytes) -> dict: + import docx + + result = {"elements": [], "errors": []} + try: + doc = docx.Document(io.BytesIO(file_bytes)) + + # Собираем все параграфы и таблицы в порядке документа + body = doc.element.body + para_idx = 0 + table_idx = 0 + + for child in body: + tag = child.tag.split("}")[-1] if "}" in child.tag else child.tag + + if tag == "p": + # Параграф + para = doc.paragraphs[para_idx] + para_idx += 1 + if para.text.strip(): + result["elements"].append({ + "type": "paragraph", + "style": para.style.name if para.style else "", + "text": para.text, + }) + + elif tag == "tbl": + # Таблица + table = doc.tables[table_idx] + table_idx += 1 + rows = [] + for row in table.rows: + cells = [cell.text for cell in row.cells] + rows.append(cells) + result["elements"].append({ + "type": "table", + "rows": rows, + }) + + except Exception as e: + result["errors"].append(f"docx parse error: {e}") + + return result + + +# ── .doc (через libreoffice) ── + +def _parse_doc(file_bytes: bytes) -> dict: + result = {"elements": [], "errors": []} + try: + with tempfile.NamedTemporaryFile(suffix=".doc", delete=False) as tmp_doc: + tmp_doc.write(file_bytes) + tmp_doc_path = tmp_doc.name + + tmp_dir = tempfile.mkdtemp() + subprocess.run( + ["libreoffice", "--headless", "--convert-to", "docx", "--outdir", tmp_dir, tmp_doc_path], + timeout=30, capture_output=True, + ) + + # Находим получившийся docx + docx_files = [f for f in os.listdir(tmp_dir) if f.endswith(".docx")] + if docx_files: + with open(os.path.join(tmp_dir, docx_files[0]), "rb") as f: + result = _parse_docx(f.read()) + else: + result["errors"].append("libreoffice conversion produced no docx") + + os.unlink(tmp_doc_path) + for f in os.listdir(tmp_dir): + os.unlink(os.path.join(tmp_dir, f)) + os.rmdir(tmp_dir) + + except FileNotFoundError: + result["errors"].append("libreoffice not installed") + except Exception as e: + result["errors"].append(f"doc parse error: {e}") + + return result + + +# ── PDF ── + +def _parse_pdf(file_bytes: bytes) -> dict: + result = {"elements": [], "errors": []} + try: + import pdfplumber + + with pdfplumber.open(io.BytesIO(file_bytes)) as pdf: + for page in pdf.pages: + # Текст страницы + text = page.extract_text() + if text: + for line in text.split("\n"): + line = line.strip() + if line: + result["elements"].append({ + "type": "paragraph", + "style": "", + "text": line, + }) + + # Таблицы на странице + tables = page.extract_tables() + for tbl in tables: + if tbl: + result["elements"].append({ + "type": "table", + "rows": tbl, + }) + + except Exception as e: + result["errors"].append(f"pdf parse error: {e}") + + return result + + +# ── ZIP ── + +def _parse_zip(file_bytes: bytes) -> dict: + result = {"elements": [], "errors": [], "files": []} + try: + with zipfile.ZipFile(io.BytesIO(file_bytes)) as zf: + for name in zf.namelist(): + if name.endswith("/"): + continue + file_data = zf.read(name) + ext = name.lower().split(".")[-1] if "." in name else "" + + mime_map = { + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "doc": "application/msword", + "pdf": "application/pdf", + } + mime = mime_map.get(ext, "") + + if mime: + parsed = parse(file_data, mime) + result["files"].append({ + "filename": name, + "elements": parsed["elements"], + "errors": parsed["errors"], + }) + else: + result["files"].append({ + "filename": name, + "elements": [], + "errors": [f"unsupported extension: {ext}"], + }) + except Exception as e: + result["errors"].append(f"zip parse error: {e}") + + return result