ElsaWin RAG pipeline: full parsing chain + ChromaDB ingest + docs

2026-06-09 06:58:57 +04:00
parent 89d2301fdf
commit 84ae6cfa9d
11 changed files with 1228 additions and 0 deletions
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Полный аудит Elsa: WI, hs2, www — какие типы, что извлекаемо."""
+import os, struct, re
+
+elsa = os.path.expanduser("~/nubes/data/elsa")
+
+# 1. WI файлы
+print("=" * 60)
+print("1. WI-файлы (OLE2 compound documents)")
+print("=" * 60)
+
+docs_rl = os.path.join(elsa, "docs", "rl")
+wi_files = [f for f in os.listdir(docs_rl) if f.endswith(".wi")]
+wi_en = [f for f in wi_files if "en-GB" in f]
+print(f"Total WI: {len(wi_files)}, en-GB: {len(wi_en)}")
+
+if wi_en:
+    path = os.path.join(docs_rl, wi_en[0])
+    with open(path, "rb") as f:
+        data = f.read(min(10000, os.path.getsize(path)))
+    
+    print(f"\nSample: {wi_en[0]}")
+    print(f"Size: {len(data)} bytes (shown), OLE2: {data[:4] == b'\xd0\xcf\x11\xe0'}")
+    print(f"Has <?xml: {b'<?xml' in data}")
+    print(f"Has <html: {b'<html' in data}")
+    
+    # Search deeper
+    for tag in [b"<?xml", b"<html", b"<body", b"<p>", b"<title"]:
+        pos = data.find(tag)
+        if pos >= 0:
+            ctx = data[pos:pos+80]
+            readable = ctx.decode("utf-16-le", errors="replace")
+            print(f"  Found '{tag.decode()}' at offset {pos}: {readable[:80]}")
+
+
+# 2. hs2 en-GB
+print("\n" + "=" * 60)
+print("2. hs2/V/en-GB/ — HTML контент")
+print("=" * 60)
+
+hs2_en = os.path.join(elsa, "docs", "hs2", "V", "en-GB")
+total_hs2 = 0
+samples = []
+for root, dirs, files in os.walk(hs2_en):
+    for f in files:
+        if f.endswith(".htm"):
+            total_hs2 += 1
+            if len(samples) < 3:
+                samples.append(os.path.join(root, f))
+
+print(f"Total HTM: {total_hs2}")
+
+for sp in samples:
+    with open(sp, "rb") as fh:
+        raw = fh.read(4096)
+        text = raw.decode("utf-16-le", errors="replace")
+        m = re.search(r"<title>(.*?)</title>", text, re.IGNORECASE)
+        title = m.group(1) if m else "(no title)"
+        body = re.sub(r"<[^>]+>", " ", text)[:200]
+        body = re.sub(r"\s+", " ", body).strip()
+        dir_name = sp.split("/")[-2]
+        print(f"  [{dir_name}] {title}")
+        print(f"    {body[:150]}")
+        print()
+
+# 3. www en-GB
+print("=" * 60)
+print("3. www/V/en-GB/ — навигация и контент")
+print("=" * 60)
+
+www_en = os.path.join(elsa, "docs", "www", "V", "en-GB")
+if os.path.exists(www_en):
+    items = os.listdir(www_en)
+    print(f"Items: {items}")
+    for item in items:
+        ipath = os.path.join(www_en, item)
+        if os.path.isfile(ipath) and item.endswith(".htm"):
+            print(f"  {item} — {os.path.getsize(ipath)} bytes")
+
+# 4. WI en-GB in other dirs
+print("\n" + "=" * 60)
+print("4. WI en-GB в других директориях")
+print("=" * 60)
+
+for subdir in ["igg", "au", "ki"]:
+    d = os.path.join(elsa, "docs", subdir)
+    if os.path.exists(d):
+        total = 0
+        en = 0
+        for f in os.listdir(d):
+            if f.endswith(".wi"):
+                total += 1
+                if "en-GB" in f:
+                    en += 1
+        print(f"  {subdir}: {total} WI total, {en} en-GB")
+
+# 5. Summary
+print("\n" + "=" * 60)
+print("ИТОГО: что можно извлечь en-GB")
+print("=" * 60)
+print("""
+MDB (уже парсено):
+  rldal.V.en-GB.mdb — 191k док-тов, иерархия ✅
+  ipsvrap.mdb — 2.5M строк, каталог запчастей ✅
+  + dbsvrfi, dbsvrfz — справочники авто (не тронуты)
+
+HTM (настоящие):
+  au/V/en-GB/ — 1295 HTM (UTF-16) — уже в elsa_jsonl ✅
+  hs2/V/en-GB/ — 4473 HTM (UTF-16) — НЕ ПАРСЕНЫ ❗
+  www/.../en-GB/ — 2538 HTM (ASCII) — НЕ ПАРСЕНЫ ❗
+  slp/V/en-GB/ — 10845 (x64 code) — МУСОР ❌
+
+WI (OLE2):
+  rl/ — 1716 en-GB WI-файлов — НЕ ПАРСЕНЫ ❗
+  igg/ — ? en-GB WI
+  au/ — ? en-GB WI
+""")