52 lines
1.6 KiB
Python
52 lines
1.6 KiB
Python
"""v2/worker.py — Фоновый парсинг через site/parser.py."""
|
|
|
|
import sys, os, json, logging, time
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'site'))
|
|
import db, parser as parser_mod, textify
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
import init as v2
|
|
|
|
logging.basicConfig(level=logging.INFO, format="[worker] %(message)s")
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def run():
|
|
logger.info("started")
|
|
while True:
|
|
try:
|
|
task = v2.pop_finalize(timeout=5)
|
|
if task:
|
|
_process(task)
|
|
except Exception as e:
|
|
logger.error("loop: %s", e)
|
|
time.sleep(1)
|
|
|
|
def _process(task):
|
|
doc_id = task.get("doc_id")
|
|
filename = task.get("filename", "")
|
|
mime = task.get("mime_type", "")
|
|
|
|
doc, err = db.query_one("SELECT original_bytes, mime_type FROM documents WHERE id=%s", (doc_id,))
|
|
if not doc:
|
|
logger.info("%s: not found", doc_id)
|
|
return
|
|
|
|
raw = doc["original_bytes"]
|
|
file_bytes = bytes(raw) if isinstance(raw, memoryview) else raw
|
|
mime = doc["mime_type"] or mime
|
|
|
|
pr = parser_mod.parse(file_bytes, mime)
|
|
elements = pr.get("elements", [])
|
|
if mime == "application/zip" and "files" in pr:
|
|
for zf in pr["files"]:
|
|
elements.extend(zf.get("elements", []))
|
|
|
|
text = textify.to_text(elements) if elements else ""
|
|
db.execute(
|
|
"UPDATE documents SET parsed_text=%s, elements_json=%s, status='parsed' WHERE id=%s",
|
|
(text, json.dumps(elements, ensure_ascii=False), doc_id),
|
|
)
|
|
logger.info("%s: parsed, %d elements", filename, len(elements))
|
|
|
|
if __name__ == "__main__":
|
|
run()
|