ElsaWin RAG pipeline: full parsing chain + ChromaDB ingest + docs

2026-06-09 06:58:57 +04:00
parent 89d2301fdf
commit 84ae6cfa9d
11 changed files with 1228 additions and 0 deletions
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""Проверить что за файлы в slp/V/en-GB/*.htm"""
+import zlib, re, os, struct
+
+path = "/home/naeel/nubes/data/elsa/docs/slp/V/en-GB/000502200001.htm"
+with open(path, "rb") as f:
+    raw = f.read()
+
+print(f"File size: {len(raw)} bytes")
+print(f"First 64 hex: {raw[:64].hex()}")
+
+# PE check
+if raw[:2] == b"MZ":
+    print(">>> It's a PE/EXE/DLL file!")
+elif raw[:2] == b"\x48\xc7":
+    print(">>> Starts with 48 c7 - could be x64 machine code!")
+
+# Look for ELF
+if raw[:4] == b"\x7fELF":
+    print(">>> It's an ELF binary!")
+
+# Try full zlib decompress at different offsets
+for offset in range(0, min(500, len(raw))):
+    if raw[offset] == 0x78 and raw[offset+1] in [0x01, 0x9c, 0xda]:
+        try:
+            decomp = zlib.decompress(raw[offset:])
+            print(f">>> zlib at offset {offset}: {len(decomp)} bytes decompressed")
+            # Try decode as UTF-16LE
+            text = decomp.decode("utf-16-le", errors="replace")
+            m = re.search(r"<title>(.*?)</title>", text, re.IGNORECASE)
+            if m:
+                print(f"    Title: {m.group(1)}")
+            body = re.sub(r"<[^>]+>", " ", text)[:500]
+            body = re.sub(r"\s+", " ", body).strip()
+            print(f"    Text: {body[:200]}")
+            break
+        except:
+            pass
+else:
+    print(">>> No zlib content found")
+
+# Also check ru-RU for comparison
+ru_path = "/home/naeel/nubes/data/elsa/docs/slp/V/ru-RU"
+if os.path.exists(ru_path):
+    ru_files = [f for f in os.listdir(ru_path) if f.endswith(".htm")]
+    if ru_files:
+        ru_file = os.path.join(ru_path, ru_files[0])
+        with open(ru_file, "rb") as f:
+            ru_raw = f.read(64)
+        print(f"\n>>> For comparison, ru-RU first 64 hex: {ru_raw.hex()}")
+        # Check encoding
+        import subprocess
+        result = subprocess.run(["file", ru_file], capture_output=True, text=True)
+        print(f"    file says: {result.stdout.strip()}")