ElsaWin RAG pipeline: full parsing chain + ChromaDB ingest + docs

2026-06-09 06:58:57 +04:00
parent 89d2301fdf
commit 84ae6cfa9d
11 changed files with 1228 additions and 0 deletions
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Парсер en-GB HTM файлов ElsaWin: hs2 + www"""
+import os, re, json, sys
+from pathlib import Path
+
+def parse_htm_file(path):
+    """Парсить один HTM-файл в UTF-16LE."""
+    with open(path, "rb") as f:
+        raw = f.read()
+    
+    # Проверяем кодировку
+    if raw[:2] == b"\xff\xfe":
+        text = raw.decode("utf-16-le", errors="replace")
+    elif raw[:2] == b"\xfe\xff":
+        text = raw.decode("utf-16-be", errors="replace")
+    else:
+        text = raw.decode("ascii", errors="replace")
+    
+    # Title
+    m = re.search(r"<title>(.*?)</title>", text, re.IGNORECASE | re.DOTALL)
+    title = m.group(1).strip() if m else ""
+    
+    # Body text
+    body = re.sub(r"<script[^>]*>.*?</script>", " ", text, flags=re.IGNORECASE | re.DOTALL)
+    body = re.sub(r"<style[^>]*>.*?</style>", " ", body, flags=re.IGNORECASE | re.DOTALL)
+    body = re.sub(r"<[^>]+>", " ", body)
+    body = re.sub(r"&nbsp;", " ", body)
+    body = re.sub(r"&\w+;", " ", body)
+    body = re.sub(r"\s+", " ", body).strip()
+    
+    return {"title": title, "text": body}
+
+
+def main():
+    if len(sys.argv) < 3:
+        print(f"Usage: {sys.argv[0]} <elsa_docs_dir> <output_dir>")
+        sys.exit(1)
+    
+    elsa_docs = os.path.abspath(sys.argv[1])
+    out_dir = os.path.abspath(sys.argv[2])
+    os.makedirs(out_dir, exist_ok=True)
+    
+    en_gb_dirs = [
+        # hs2 — in nested structure: hs2/V/en-GB/src/XX/XXXXXX/master.htm
+        os.path.join(elsa_docs, "hs2", "V", "en-GB"),
+        # www — various paths under www/.../en-GB/
+    ]
+    
+    # Find all en-GB HTM
+    all_files = []
+    for root, dirs, files in os.walk(elsa_docs):
+        if "/en-GB/" in root and not "/slp/" in root:
+            for f in files:
+                if f.endswith(".htm") or f.endswith(".html"):
+                    all_files.append(os.path.join(root, f))
+    
+    # Skip already-known-malformed (slp — machine code, already excluded above)
+    # Only au, hs2, www, igg
+    good_files = [f for f in all_files if any(d in f for d in ["/www/", "/hs2/", "/au/"])]
+    
+    print(f"Found {len(good_files)} en-GB HTM files (excluding slp)")
+    
+    # Split by source
+    by_source = {}
+    for f in good_files:
+        if "/au/" in f:
+            src = "au"
+        elif "/hs2/" in f:
+            src = "hs2"
+        elif "/www/" in f:
+            src = "www"
+        else:
+            src = "other"
+        by_source.setdefault(src, []).append(f)
+    
+    for src, files in by_source.items():
+        print(f"  {src}: {len(files)} files")
+    
+    # Parse and save
+    total = 0
+    part_size = 5000
+    part_idx = 0
+    buf = []
+    out_file = None
+    
+    for src, files in by_source.items():
+        for fpath in sorted(files):
+            try:
+                doc = parse_htm_file(fpath)
+                rel = os.path.relpath(fpath, elsa_docs)
+                record = {
+                    "source": f"elsa_{src}",
+                    "file": "/docs/" + rel,
+                    "title": doc["title"],
+                    "text": doc["text"],
+                }
+                buf.append(record)
+                total += 1
+                
+                if len(buf) >= part_size:
+                    out_file = os.path.join(out_dir, f"part_{part_idx:04d}.jsonl")
+                    with open(out_file, "w", encoding="utf-8") as f:
+                        for r in buf:
+                            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+                    print(f"  Part {part_idx:04d}: {len(buf)} records → {out_file}")
+                    buf = []
+                    part_idx += 1
+            except Exception as e:
+                print(f"  ERROR {fpath}: {e}", file=sys.stderr)
+    
+    # Flush remaining
+    if buf:
+        out_file = os.path.join(out_dir, f"part_{part_idx:04d}.jsonl")
+        with open(out_file, "w", encoding="utf-8") as f:
+            for r in buf:
+                f.write(json.dumps(r, ensure_ascii=False) + "\n")
+        print(f"  Part {part_idx:04d}: {len(buf)} records → {out_file}")
+        part_idx += 1
+    
+    print(f"\nTotal: {total} documents in {part_idx} parts → {out_dir}")
+
+
+if __name__ == "__main__":
+    main()