ElsaWin RAG pipeline: full parsing chain + ChromaDB ingest + docs
This commit is contained in:
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Полный аудит Elsa: WI, hs2, www — какие типы, что извлекаемо."""
|
||||
import os, struct, re
|
||||
|
||||
elsa = os.path.expanduser("~/nubes/data/elsa")
|
||||
|
||||
# 1. WI файлы
|
||||
print("=" * 60)
|
||||
print("1. WI-файлы (OLE2 compound documents)")
|
||||
print("=" * 60)
|
||||
|
||||
docs_rl = os.path.join(elsa, "docs", "rl")
|
||||
wi_files = [f for f in os.listdir(docs_rl) if f.endswith(".wi")]
|
||||
wi_en = [f for f in wi_files if "en-GB" in f]
|
||||
print(f"Total WI: {len(wi_files)}, en-GB: {len(wi_en)}")
|
||||
|
||||
if wi_en:
|
||||
path = os.path.join(docs_rl, wi_en[0])
|
||||
with open(path, "rb") as f:
|
||||
data = f.read(min(10000, os.path.getsize(path)))
|
||||
|
||||
print(f"\nSample: {wi_en[0]}")
|
||||
print(f"Size: {len(data)} bytes (shown), OLE2: {data[:4] == b'\xd0\xcf\x11\xe0'}")
|
||||
print(f"Has <?xml: {b'<?xml' in data}")
|
||||
print(f"Has <html: {b'<html' in data}")
|
||||
|
||||
# Search deeper
|
||||
for tag in [b"<?xml", b"<html", b"<body", b"<p>", b"<title"]:
|
||||
pos = data.find(tag)
|
||||
if pos >= 0:
|
||||
ctx = data[pos:pos+80]
|
||||
readable = ctx.decode("utf-16-le", errors="replace")
|
||||
print(f" Found '{tag.decode()}' at offset {pos}: {readable[:80]}")
|
||||
|
||||
|
||||
# 2. hs2 en-GB
|
||||
print("\n" + "=" * 60)
|
||||
print("2. hs2/V/en-GB/ — HTML контент")
|
||||
print("=" * 60)
|
||||
|
||||
hs2_en = os.path.join(elsa, "docs", "hs2", "V", "en-GB")
|
||||
total_hs2 = 0
|
||||
samples = []
|
||||
for root, dirs, files in os.walk(hs2_en):
|
||||
for f in files:
|
||||
if f.endswith(".htm"):
|
||||
total_hs2 += 1
|
||||
if len(samples) < 3:
|
||||
samples.append(os.path.join(root, f))
|
||||
|
||||
print(f"Total HTM: {total_hs2}")
|
||||
|
||||
for sp in samples:
|
||||
with open(sp, "rb") as fh:
|
||||
raw = fh.read(4096)
|
||||
text = raw.decode("utf-16-le", errors="replace")
|
||||
m = re.search(r"<title>(.*?)</title>", text, re.IGNORECASE)
|
||||
title = m.group(1) if m else "(no title)"
|
||||
body = re.sub(r"<[^>]+>", " ", text)[:200]
|
||||
body = re.sub(r"\s+", " ", body).strip()
|
||||
dir_name = sp.split("/")[-2]
|
||||
print(f" [{dir_name}] {title}")
|
||||
print(f" {body[:150]}")
|
||||
print()
|
||||
|
||||
# 3. www en-GB
|
||||
print("=" * 60)
|
||||
print("3. www/V/en-GB/ — навигация и контент")
|
||||
print("=" * 60)
|
||||
|
||||
www_en = os.path.join(elsa, "docs", "www", "V", "en-GB")
|
||||
if os.path.exists(www_en):
|
||||
items = os.listdir(www_en)
|
||||
print(f"Items: {items}")
|
||||
for item in items:
|
||||
ipath = os.path.join(www_en, item)
|
||||
if os.path.isfile(ipath) and item.endswith(".htm"):
|
||||
print(f" {item} — {os.path.getsize(ipath)} bytes")
|
||||
|
||||
# 4. WI en-GB in other dirs
|
||||
print("\n" + "=" * 60)
|
||||
print("4. WI en-GB в других директориях")
|
||||
print("=" * 60)
|
||||
|
||||
for subdir in ["igg", "au", "ki"]:
|
||||
d = os.path.join(elsa, "docs", subdir)
|
||||
if os.path.exists(d):
|
||||
total = 0
|
||||
en = 0
|
||||
for f in os.listdir(d):
|
||||
if f.endswith(".wi"):
|
||||
total += 1
|
||||
if "en-GB" in f:
|
||||
en += 1
|
||||
print(f" {subdir}: {total} WI total, {en} en-GB")
|
||||
|
||||
# 5. Summary
|
||||
print("\n" + "=" * 60)
|
||||
print("ИТОГО: что можно извлечь en-GB")
|
||||
print("=" * 60)
|
||||
print("""
|
||||
MDB (уже парсено):
|
||||
rldal.V.en-GB.mdb — 191k док-тов, иерархия ✅
|
||||
ipsvrap.mdb — 2.5M строк, каталог запчастей ✅
|
||||
+ dbsvrfi, dbsvrfz — справочники авто (не тронуты)
|
||||
|
||||
HTM (настоящие):
|
||||
au/V/en-GB/ — 1295 HTM (UTF-16) — уже в elsa_jsonl ✅
|
||||
hs2/V/en-GB/ — 4473 HTM (UTF-16) — НЕ ПАРСЕНЫ ❗
|
||||
www/.../en-GB/ — 2538 HTM (ASCII) — НЕ ПАРСЕНЫ ❗
|
||||
slp/V/en-GB/ — 10845 (x64 code) — МУСОР ❌
|
||||
|
||||
WI (OLE2):
|
||||
rl/ — 1716 en-GB WI-файлов — НЕ ПАРСЕНЫ ❗
|
||||
igg/ — ? en-GB WI
|
||||
au/ — ? en-GB WI
|
||||
""")
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверить что за файлы в slp/V/en-GB/*.htm"""
|
||||
import zlib, re, os, struct
|
||||
|
||||
path = "/home/naeel/nubes/data/elsa/docs/slp/V/en-GB/000502200001.htm"
|
||||
with open(path, "rb") as f:
|
||||
raw = f.read()
|
||||
|
||||
print(f"File size: {len(raw)} bytes")
|
||||
print(f"First 64 hex: {raw[:64].hex()}")
|
||||
|
||||
# PE check
|
||||
if raw[:2] == b"MZ":
|
||||
print(">>> It's a PE/EXE/DLL file!")
|
||||
elif raw[:2] == b"\x48\xc7":
|
||||
print(">>> Starts with 48 c7 - could be x64 machine code!")
|
||||
|
||||
# Look for ELF
|
||||
if raw[:4] == b"\x7fELF":
|
||||
print(">>> It's an ELF binary!")
|
||||
|
||||
# Try full zlib decompress at different offsets
|
||||
for offset in range(0, min(500, len(raw))):
|
||||
if raw[offset] == 0x78 and raw[offset+1] in [0x01, 0x9c, 0xda]:
|
||||
try:
|
||||
decomp = zlib.decompress(raw[offset:])
|
||||
print(f">>> zlib at offset {offset}: {len(decomp)} bytes decompressed")
|
||||
# Try decode as UTF-16LE
|
||||
text = decomp.decode("utf-16-le", errors="replace")
|
||||
m = re.search(r"<title>(.*?)</title>", text, re.IGNORECASE)
|
||||
if m:
|
||||
print(f" Title: {m.group(1)}")
|
||||
body = re.sub(r"<[^>]+>", " ", text)[:500]
|
||||
body = re.sub(r"\s+", " ", body).strip()
|
||||
print(f" Text: {body[:200]}")
|
||||
break
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
print(">>> No zlib content found")
|
||||
|
||||
# Also check ru-RU for comparison
|
||||
ru_path = "/home/naeel/nubes/data/elsa/docs/slp/V/ru-RU"
|
||||
if os.path.exists(ru_path):
|
||||
ru_files = [f for f in os.listdir(ru_path) if f.endswith(".htm")]
|
||||
if ru_files:
|
||||
ru_file = os.path.join(ru_path, ru_files[0])
|
||||
with open(ru_file, "rb") as f:
|
||||
ru_raw = f.read(64)
|
||||
print(f"\n>>> For comparison, ru-RU first 64 hex: {ru_raw.hex()}")
|
||||
# Check encoding
|
||||
import subprocess
|
||||
result = subprocess.run(["file", ru_file], capture_output=True, text=True)
|
||||
print(f" file says: {result.stdout.strip()}")
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Парсер en-GB HTM файлов ElsaWin: hs2 + www"""
|
||||
import os, re, json, sys
|
||||
from pathlib import Path
|
||||
|
||||
def parse_htm_file(path):
|
||||
"""Парсить один HTM-файл в UTF-16LE."""
|
||||
with open(path, "rb") as f:
|
||||
raw = f.read()
|
||||
|
||||
# Проверяем кодировку
|
||||
if raw[:2] == b"\xff\xfe":
|
||||
text = raw.decode("utf-16-le", errors="replace")
|
||||
elif raw[:2] == b"\xfe\xff":
|
||||
text = raw.decode("utf-16-be", errors="replace")
|
||||
else:
|
||||
text = raw.decode("ascii", errors="replace")
|
||||
|
||||
# Title
|
||||
m = re.search(r"<title>(.*?)</title>", text, re.IGNORECASE | re.DOTALL)
|
||||
title = m.group(1).strip() if m else ""
|
||||
|
||||
# Body text
|
||||
body = re.sub(r"<script[^>]*>.*?</script>", " ", text, flags=re.IGNORECASE | re.DOTALL)
|
||||
body = re.sub(r"<style[^>]*>.*?</style>", " ", body, flags=re.IGNORECASE | re.DOTALL)
|
||||
body = re.sub(r"<[^>]+>", " ", body)
|
||||
body = re.sub(r" ", " ", body)
|
||||
body = re.sub(r"&\w+;", " ", body)
|
||||
body = re.sub(r"\s+", " ", body).strip()
|
||||
|
||||
return {"title": title, "text": body}
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Usage: {sys.argv[0]} <elsa_docs_dir> <output_dir>")
|
||||
sys.exit(1)
|
||||
|
||||
elsa_docs = os.path.abspath(sys.argv[1])
|
||||
out_dir = os.path.abspath(sys.argv[2])
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
en_gb_dirs = [
|
||||
# hs2 — in nested structure: hs2/V/en-GB/src/XX/XXXXXX/master.htm
|
||||
os.path.join(elsa_docs, "hs2", "V", "en-GB"),
|
||||
# www — various paths under www/.../en-GB/
|
||||
]
|
||||
|
||||
# Find all en-GB HTM
|
||||
all_files = []
|
||||
for root, dirs, files in os.walk(elsa_docs):
|
||||
if "/en-GB/" in root and not "/slp/" in root:
|
||||
for f in files:
|
||||
if f.endswith(".htm") or f.endswith(".html"):
|
||||
all_files.append(os.path.join(root, f))
|
||||
|
||||
# Skip already-known-malformed (slp — machine code, already excluded above)
|
||||
# Only au, hs2, www, igg
|
||||
good_files = [f for f in all_files if any(d in f for d in ["/www/", "/hs2/", "/au/"])]
|
||||
|
||||
print(f"Found {len(good_files)} en-GB HTM files (excluding slp)")
|
||||
|
||||
# Split by source
|
||||
by_source = {}
|
||||
for f in good_files:
|
||||
if "/au/" in f:
|
||||
src = "au"
|
||||
elif "/hs2/" in f:
|
||||
src = "hs2"
|
||||
elif "/www/" in f:
|
||||
src = "www"
|
||||
else:
|
||||
src = "other"
|
||||
by_source.setdefault(src, []).append(f)
|
||||
|
||||
for src, files in by_source.items():
|
||||
print(f" {src}: {len(files)} files")
|
||||
|
||||
# Parse and save
|
||||
total = 0
|
||||
part_size = 5000
|
||||
part_idx = 0
|
||||
buf = []
|
||||
out_file = None
|
||||
|
||||
for src, files in by_source.items():
|
||||
for fpath in sorted(files):
|
||||
try:
|
||||
doc = parse_htm_file(fpath)
|
||||
rel = os.path.relpath(fpath, elsa_docs)
|
||||
record = {
|
||||
"source": f"elsa_{src}",
|
||||
"file": "/docs/" + rel,
|
||||
"title": doc["title"],
|
||||
"text": doc["text"],
|
||||
}
|
||||
buf.append(record)
|
||||
total += 1
|
||||
|
||||
if len(buf) >= part_size:
|
||||
out_file = os.path.join(out_dir, f"part_{part_idx:04d}.jsonl")
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
for r in buf:
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
print(f" Part {part_idx:04d}: {len(buf)} records → {out_file}")
|
||||
buf = []
|
||||
part_idx += 1
|
||||
except Exception as e:
|
||||
print(f" ERROR {fpath}: {e}", file=sys.stderr)
|
||||
|
||||
# Flush remaining
|
||||
if buf:
|
||||
out_file = os.path.join(out_dir, f"part_{part_idx:04d}.jsonl")
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
for r in buf:
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
print(f" Part {part_idx:04d}: {len(buf)} records → {out_file}")
|
||||
part_idx += 1
|
||||
|
||||
print(f"\nTotal: {total} documents in {part_idx} parts → {out_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Парсер MDB-баз ElsaWin → JSONL для RAG.
|
||||
Использует mdbtools (mdb-export) для выгрузки таблиц.
|
||||
|
||||
Использование:
|
||||
python3 parse_elsa_mdb.py <путь_к_mdb> <выходная_директория>
|
||||
|
||||
Пример:
|
||||
python3 parse_elsa_mdb.py ~/nubes/data/elsa/data/rldal.V.en-GB.mdb ~/nubes/data/elsa_mdb_jsonl/
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def mdb_tables(mdb_path: str) -> list[str]:
|
||||
"""Получить список таблиц из MDB через mdb-tables."""
|
||||
result = subprocess.run(
|
||||
["mdb-tables", mdb_path],
|
||||
capture_output=True, text=True, check=True
|
||||
)
|
||||
return result.stdout.strip().split()
|
||||
|
||||
|
||||
def mdb_export(mdb_path: str, table: str) -> str:
|
||||
"""Экспорт таблицы в CSV через mdb-export."""
|
||||
result = subprocess.run(
|
||||
["mdb-export", mdb_path, table],
|
||||
capture_output=True, text=True, check=True
|
||||
)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def parse_csv(csv_text: str) -> list[dict]:
|
||||
"""Распарсить CSV в список словарей."""
|
||||
reader = csv.DictReader(io.StringIO(csv_text))
|
||||
return [row for row in reader]
|
||||
|
||||
|
||||
def export_table_to_jsonl(mdb_path: str, table: str, output_dir: str) -> str:
|
||||
"""Экспорт одной таблицы в JSONL."""
|
||||
print(f" {table}...", end=" ", flush=True)
|
||||
csv_text = mdb_export(mdb_path, table)
|
||||
rows = parse_csv(csv_text)
|
||||
|
||||
out_file = os.path.join(output_dir, f"{table}.jsonl")
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
for row in rows:
|
||||
# Очистка значений: убираем лишние пробелы
|
||||
cleaned = {k: v.strip() if v else "" for k, v in row.items()}
|
||||
f.write(json.dumps(cleaned, ensure_ascii=False) + "\n")
|
||||
|
||||
print(f"{len(rows)} rows → {out_file}")
|
||||
return out_file
|
||||
|
||||
|
||||
def build_wi_dokument_docs(mdb_path: str, output_dir: str):
|
||||
"""
|
||||
Построить документы из wi_dokument с полной иерархией.
|
||||
Каждый документ = одна строка JSONL с полным путём разделов.
|
||||
"""
|
||||
print("\n=== Построение документов из wi_dokument ===")
|
||||
|
||||
csv_text = mdb_export(mdb_path, "wi_dokument")
|
||||
rows = parse_csv(csv_text)
|
||||
|
||||
docs = []
|
||||
for row in rows:
|
||||
# Собираем полный путь иерархии
|
||||
hierarchy_parts = []
|
||||
for key in ["og_bez", "bg_bez", "rg_bez", "hkap_bez", "kap_bez", "ukap_bez"]:
|
||||
val = row.get(key, "").strip()
|
||||
if val:
|
||||
hierarchy_parts.append(val)
|
||||
|
||||
full_path = " → ".join(hierarchy_parts) if hierarchy_parts else ""
|
||||
|
||||
doc = {
|
||||
"dokument_id": int(row.get("dokument_id", 0)),
|
||||
"spk": row.get("spk", "").strip(),
|
||||
"redsystyp": int(row.get("redsystyp", 0)),
|
||||
"name": row.get("name", "").strip(),
|
||||
"hierarchy": {
|
||||
"og_id": int(row.get("og_id", 0)) if row.get("og_id", "").strip() else None,
|
||||
"bg_id": int(row.get("bg_id", 0)) if row.get("bg_id", "").strip() else None,
|
||||
"rg_id": row.get("rg_id", "").strip(),
|
||||
"hkap_id": int(row.get("hkap_id", 0)) if row.get("hkap_id", "").strip() else None,
|
||||
"kap_id": int(row.get("kap_id", 0)) if row.get("kap_id", "").strip() else None,
|
||||
"ukap_id": int(row.get("ukap_id", 0)) if row.get("ukap_id", "").strip() else None,
|
||||
},
|
||||
"titles": {
|
||||
"og": row.get("og_bez", "").strip(),
|
||||
"bg": row.get("bg_bez", "").strip(),
|
||||
"rg": row.get("rg_bez", "").strip(),
|
||||
"hkap": row.get("hkap_bez", "").strip(),
|
||||
"kap": row.get("kap_bez", "").strip(),
|
||||
"ukap": row.get("ukap_bez", "").strip(),
|
||||
},
|
||||
"full_title": full_path,
|
||||
"parent_dokument_id": int(row.get("parent_dokument_id", 0)) if row.get("parent_dokument_id", "").strip() else None,
|
||||
"ext_id": row.get("ext_id", "").strip(),
|
||||
"source": "elsa",
|
||||
"type": "wi_dokument",
|
||||
}
|
||||
docs.append(doc)
|
||||
|
||||
out_file = os.path.join(output_dir, "wi_dokument_enriched.jsonl")
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
for doc in docs:
|
||||
f.write(json.dumps(doc, ensure_ascii=False) + "\n")
|
||||
|
||||
print(f"Построено {len(docs)} документов → {out_file}")
|
||||
|
||||
# Статистика по иерархии
|
||||
with_titles = sum(1 for d in docs if d["full_title"])
|
||||
print(f" Из них с иерархией: {with_titles}")
|
||||
print(f" Без иерархии: {len(docs) - with_titles}")
|
||||
|
||||
return out_file
|
||||
|
||||
|
||||
def build_vehicle_lookup(mdb_path: str, output_dir: str):
|
||||
"""
|
||||
Построить справочник привязки документов к автомобилям.
|
||||
"""
|
||||
print("\n=== Построение привязки документов к автомобилям ===")
|
||||
|
||||
# wi_dokument_fzg
|
||||
csv_text = mdb_export(mdb_path, "wi_dokument_fzg")
|
||||
rows = parse_csv(csv_text)
|
||||
|
||||
# Группируем по dokument_id
|
||||
from collections import defaultdict
|
||||
by_doc = defaultdict(list)
|
||||
for row in rows:
|
||||
entry = {
|
||||
"vtyp": row.get("vtyp", "").strip(),
|
||||
"marke": row.get("marke", "").strip(),
|
||||
"mkb": row.get("mkb", "").strip(),
|
||||
"gkb": row.get("gkb", "").strip(),
|
||||
"gtyp": row.get("gtyp", "").strip(),
|
||||
"mj_von": int(row.get("mj_von", 0)) if row.get("mj_von", "").strip() else None,
|
||||
"mj_bis": int(row.get("mj_bis", 0)) if row.get("mj_bis", "").strip() else None,
|
||||
}
|
||||
by_doc[int(row["dokument_id"])].append(entry)
|
||||
|
||||
out_file = os.path.join(output_dir, "wi_dokument_fzg_grouped.jsonl")
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
for doc_id, vehicles in by_doc.items():
|
||||
f.write(json.dumps({
|
||||
"dokument_id": doc_id,
|
||||
"vehicles": vehicles,
|
||||
"count": len(vehicles),
|
||||
}, ensure_ascii=False) + "\n")
|
||||
|
||||
print(f"{len(by_doc)} документов с привязкой к авто → {out_file}")
|
||||
return out_file
|
||||
|
||||
|
||||
def build_hierarchy_reference(mdb_path: str, output_dir: str):
|
||||
"""
|
||||
Построить справочники иерархии: og, bg, rg с vehicle-привязкой.
|
||||
"""
|
||||
print("\n=== Построение справочников иерархии ===")
|
||||
|
||||
for tbl, name in [("wi_og_fzg", "og"), ("wi_bg_fzg", "bg"), ("wi_rg_fzg", "rg")]:
|
||||
csv_text = mdb_export(mdb_path, tbl)
|
||||
rows = parse_csv(csv_text)
|
||||
out_file = os.path.join(output_dir, f"{tbl}.jsonl")
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
for row in rows:
|
||||
cleaned = {k: v.strip() if v else "" for k, v in row.items()}
|
||||
f.write(json.dumps(cleaned, ensure_ascii=False) + "\n")
|
||||
print(f" {tbl}: {len(rows)} rows → {out_file}")
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
print(__doc__)
|
||||
sys.exit(1)
|
||||
|
||||
mdb_path = os.path.abspath(sys.argv[1])
|
||||
output_dir = os.path.abspath(sys.argv[2])
|
||||
|
||||
if not os.path.exists(mdb_path):
|
||||
print(f"ОШИБКА: MDB файл не найден: {mdb_path}")
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
print(f"MDB: {mdb_path}")
|
||||
print(f"Выход: {output_dir}")
|
||||
|
||||
# 1. Список таблиц
|
||||
tables = mdb_tables(mdb_path)
|
||||
print(f"\nТаблицы ({len(tables)}): {', '.join(tables)}")
|
||||
|
||||
# 2. Экспорт каждой таблицы в сырой JSONL
|
||||
print("\n=== Экспорт таблиц ===")
|
||||
for table in tables:
|
||||
try:
|
||||
export_table_to_jsonl(mdb_path, table, output_dir)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"ОШИБКА экспорта {table}: {e}")
|
||||
|
||||
# 3. Построение обогащённых документов (rldal-specific)
|
||||
if "wi_dokument" in tables:
|
||||
build_wi_dokument_docs(mdb_path, output_dir)
|
||||
build_vehicle_lookup(mdb_path, output_dir)
|
||||
build_hierarchy_reference(mdb_path, output_dir)
|
||||
else:
|
||||
print("\n=== Пропускаем обогащение (нет wi_dokument таблицы) ===")
|
||||
|
||||
print(f"\n✅ Готово. Результат: {output_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Парсер WI-файлов ElsaWin (OLE2 → XML → JSONL)"""
|
||||
import os, re, json, sys
|
||||
import olefile
|
||||
|
||||
def parse_wi_file(path):
|
||||
"""Извлечь XML-потоки из WI (OLE2) файла."""
|
||||
docs = []
|
||||
|
||||
with olefile.OleFileIO(path) as ole:
|
||||
for stream in ole.listdir():
|
||||
stream_name = "/".join(stream)
|
||||
if not stream_name.endswith(".xml"):
|
||||
continue
|
||||
|
||||
data = ole.openstream(stream).read()
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
|
||||
# Extract key elements
|
||||
title_parts = []
|
||||
|
||||
m = re.search(r"<marke>(.*?)</marke>", text, re.IGNORECASE)
|
||||
marke = m.group(1) if m else ""
|
||||
|
||||
m = re.search(r"<modell>(.*?)</modell>", text, re.IGNORECASE)
|
||||
modell = m.group(1) if m else ""
|
||||
|
||||
m = re.search(r"<obergrup>(.*?)</obergrup>", text, re.IGNORECASE)
|
||||
obergrup = m.group(1) if m else ""
|
||||
|
||||
m = re.search(r"<baugrup>(.*?)</baugrup>", text, re.IGNORECASE)
|
||||
baugrup = m.group(1) if m else ""
|
||||
|
||||
# Название документа
|
||||
m = re.search(r"<h-kap[^>]*?typen-bez=\"([^\"]+)\"", text)
|
||||
typen = m.group(1) if m else ""
|
||||
|
||||
# Title from vorspann
|
||||
title = " ".join(p for p in [obergrup, baugrup] if p)
|
||||
if marke:
|
||||
title = f"{marke} {modell}: {title}" if modell else f"{marke}: {title}"
|
||||
|
||||
# Clean body
|
||||
body = re.sub(r"<[^>]+>", " ", text)
|
||||
body = re.sub(r"\s+", " ", body).strip()
|
||||
|
||||
docs.append({
|
||||
"title": title,
|
||||
"marke": marke,
|
||||
"modell": modell,
|
||||
"obergrup": obergrup,
|
||||
"baugrup": baugrup,
|
||||
"text": body,
|
||||
})
|
||||
|
||||
return docs
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 3:
|
||||
print(f"Usage: {sys.argv[0]} <elsa_docs_dir> <output_dir>")
|
||||
sys.exit(1)
|
||||
|
||||
elsa_docs = os.path.abspath(sys.argv[1])
|
||||
out_dir = os.path.abspath(sys.argv[2])
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
# Collect en-GB WI files
|
||||
wi_files = []
|
||||
for root, dirs, files in os.walk(elsa_docs):
|
||||
for f in files:
|
||||
if f.endswith(".wi") and "en-GB" in f:
|
||||
wi_files.append(os.path.join(root, f))
|
||||
|
||||
print(f"Found {len(wi_files)} en-GB WI files")
|
||||
|
||||
# By category
|
||||
by_cat = {}
|
||||
for f in wi_files:
|
||||
if "/rl/" in f:
|
||||
cat = "rl"
|
||||
elif "/igg/" in f:
|
||||
cat = "igg"
|
||||
elif "/ki/" in f:
|
||||
cat = "ki"
|
||||
else:
|
||||
cat = "other"
|
||||
by_cat.setdefault(cat, []).append(f)
|
||||
|
||||
for cat, files in by_cat.items():
|
||||
print(f" {cat}: {len(files)} files")
|
||||
|
||||
# Parse
|
||||
total_docs = 0
|
||||
part_size = 5000
|
||||
part_idx = 0
|
||||
buf = []
|
||||
|
||||
for cat, files in by_cat.items():
|
||||
for fpath in sorted(files):
|
||||
try:
|
||||
docs = parse_wi_file(fpath)
|
||||
for doc in docs:
|
||||
doc["source"] = f"elsa_wi_{cat}"
|
||||
doc["file"] = os.path.relpath(fpath, elsa_docs)
|
||||
buf.append(doc)
|
||||
total_docs += 1
|
||||
except Exception as e:
|
||||
print(f" ERROR {fpath}: {e}", file=sys.stderr)
|
||||
|
||||
if len(buf) >= part_size:
|
||||
out_file = os.path.join(out_dir, f"part_{part_idx:04d}.jsonl")
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
for r in buf:
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
print(f" Part {part_idx:04d}: {len(buf)} → {out_file}")
|
||||
buf = []
|
||||
part_idx += 1
|
||||
|
||||
if buf:
|
||||
out_file = os.path.join(out_dir, f"part_{part_idx:04d}.jsonl")
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
for r in buf:
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
print(f" Part {part_idx:04d}: {len(buf)} → {out_file}")
|
||||
|
||||
print(f"\nTotal: {total_docs} XML documents from {len(wi_files)} WI files → {out_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ingest JSONL into ChromaDB — uses llama.cpp /embedding endpoint."""
|
||||
import json, os, sys, glob, urllib.request, time
|
||||
import chromadb
|
||||
|
||||
LLAMA_URL = "http://localhost:8081/embedding"
|
||||
|
||||
def llama_embed(texts):
|
||||
"""Get embeddings from nomic-embed server with retry."""
|
||||
data = json.dumps({"content": texts}).encode()
|
||||
req = urllib.request.Request(LLAMA_URL, data=data,
|
||||
headers={"Content-Type": "application/json"})
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
body = json.loads(resp.read())
|
||||
if isinstance(body, list):
|
||||
return [item["embedding"][0] for item in body]
|
||||
return body.get("embedding", [])
|
||||
except Exception as e:
|
||||
if attempt < 2:
|
||||
print(f"[retry {attempt+1}] {e}")
|
||||
time.sleep(5)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
class LlamaEmbedFn:
|
||||
"""Custom embedding function using llama.cpp."""
|
||||
def __call__(self, input):
|
||||
all_embs = []
|
||||
for i in range(0, len(input), 32):
|
||||
batch = input[i:i+32]
|
||||
embs = llama_embed(batch)
|
||||
if isinstance(embs[0], float):
|
||||
all_embs.append(embs)
|
||||
else:
|
||||
all_embs.extend(embs)
|
||||
return all_embs
|
||||
|
||||
|
||||
def main():
|
||||
jsonl_dir = sys.argv[1] if len(sys.argv) > 1 else os.path.expanduser("~/nubes/data/elsa_jsonl_wi_en")
|
||||
db_dir = sys.argv[2] if len(sys.argv) > 2 else os.path.expanduser("~/nubes/chroma_db")
|
||||
collection_name = sys.argv[3] if len(sys.argv) > 3 else "elsa_docs"
|
||||
|
||||
ef = LlamaEmbedFn()
|
||||
|
||||
client = chromadb.PersistentClient(path=db_dir)
|
||||
|
||||
try:
|
||||
client.delete_collection(collection_name)
|
||||
except:
|
||||
pass
|
||||
|
||||
collection = client.create_collection(
|
||||
name=collection_name,
|
||||
embedding_function=ef,
|
||||
metadata={"hnsw:space": "cosine"}
|
||||
)
|
||||
|
||||
jsonl_files = sorted(glob.glob(os.path.join(jsonl_dir, "*.jsonl")))
|
||||
print(f"Found {len(jsonl_files)} JSONL files")
|
||||
print(f"Using llama.cpp at {LLAMA_URL}")
|
||||
|
||||
total = 0
|
||||
for jf in jsonl_files:
|
||||
ids, docs, metas = [], [], []
|
||||
|
||||
with open(jf, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
try:
|
||||
d = json.loads(line)
|
||||
title = d.get("title", "").strip()
|
||||
text = d.get("text", "").strip()
|
||||
ft = d.get("full_title", "").strip()
|
||||
|
||||
if ft and text:
|
||||
doc = f"{ft}\n{text}"
|
||||
elif title and text:
|
||||
doc = f"{title}\n{text}"
|
||||
else:
|
||||
doc = text or title or ft
|
||||
|
||||
if not doc or len(doc) < 20:
|
||||
continue
|
||||
|
||||
# Split long docs into chunks of ~1000 chars
|
||||
chunks = [doc[i:i+1000] for i in range(0, len(doc), 1000)]
|
||||
for chunk in chunks:
|
||||
if len(chunk) < 20:
|
||||
continue
|
||||
ids.append(str(total))
|
||||
docs.append(chunk)
|
||||
metas.append({
|
||||
"source": d.get("source", ""),
|
||||
"file": d.get("file", ""),
|
||||
"title": title or ft or doc[:100],
|
||||
})
|
||||
total += 1
|
||||
except:
|
||||
continue
|
||||
|
||||
if ids:
|
||||
# Send in smaller batches of 500
|
||||
for i in range(0, len(ids), 500):
|
||||
batch_ids = ids[i:i+500]
|
||||
batch_docs = docs[i:i+500]
|
||||
batch_metas = metas[i:i+500]
|
||||
for attempt in range(3):
|
||||
try:
|
||||
collection.add(ids=batch_ids, documents=batch_docs, metadatas=batch_metas)
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt < 2:
|
||||
print(f" [retry {os.path.basename(jf)} batch {attempt+1}] {e}")
|
||||
time.sleep(10)
|
||||
else:
|
||||
print(f" [SKIP batch] {e}")
|
||||
print(f" {os.path.basename(jf)}: +{len(ids)} (total {total})")
|
||||
|
||||
print(f"\nDone. {total} documents → {db_dir}/{collection_name}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Полная индексация через sentence-transformers (CPU, стабильно)."""
|
||||
import json, os, sys, time, glob, shutil
|
||||
from chromadb import PersistentClient
|
||||
from chromadb.utils import embedding_functions
|
||||
|
||||
ALL_DIRS = [
|
||||
"~/nubes/data/elsa_jsonl_wi_en",
|
||||
"~/nubes/data/elsa_mdb_jsonl",
|
||||
"~/nubes/data/elsa_mdb_jsonl/ipsvrap",
|
||||
"~/nubes/data/elsa_jsonl_htm_en",
|
||||
"~/nubes/data/elsa_jsonl",
|
||||
"~/nubes/data/elsa_mdb_jsonl/dbsvrfi",
|
||||
"~/nubes/data/elsa_mdb_jsonl/dbsvrfz",
|
||||
]
|
||||
|
||||
DB_DIR = os.path.expanduser("~/nubes/chroma_db")
|
||||
MODEL = "all-MiniLM-L6-v2" # 90 MB, CPU
|
||||
|
||||
ef = embedding_functions.SentenceTransformerEmbeddingFunction(model_name=MODEL)
|
||||
|
||||
if os.path.exists(DB_DIR):
|
||||
shutil.rmtree(DB_DIR)
|
||||
print("Old chroma_db deleted")
|
||||
|
||||
client = PersistentClient(path=DB_DIR)
|
||||
collection = client.create_collection("elsa_docs", embedding_function=ef)
|
||||
|
||||
all_files = []
|
||||
for d in ALL_DIRS:
|
||||
path = os.path.expanduser(d)
|
||||
files = sorted(glob.glob(os.path.join(path, "*.jsonl")))
|
||||
all_files.extend(files)
|
||||
print(f"{d}: {len(files)} files")
|
||||
|
||||
print(f"\nTotal: {len(all_files)} JSONL files")
|
||||
|
||||
total = 0
|
||||
for fpath in all_files:
|
||||
docs, ids, metas = [], [], []
|
||||
prefix = os.path.basename(os.path.dirname(fpath)) + "_"
|
||||
|
||||
try:
|
||||
with open(fpath) as fh:
|
||||
for line in fh:
|
||||
try:
|
||||
d = json.loads(line)
|
||||
title = d.get("title", "") or d.get("full_title", "")
|
||||
text = d.get("text", "")
|
||||
doc = (title + "\n" + text).strip()[:1000]
|
||||
if len(doc) < 20:
|
||||
continue
|
||||
docs.append(doc)
|
||||
ids.append(f"{prefix}{os.path.basename(fpath)}_{len(docs)}")
|
||||
metas.append({"source": prefix.strip("_")})
|
||||
except:
|
||||
continue
|
||||
except:
|
||||
continue
|
||||
|
||||
if not docs:
|
||||
continue
|
||||
|
||||
print(f"{os.path.basename(fpath)}: {len(docs)} docs", end="", flush=True)
|
||||
|
||||
for j in range(0, len(docs), 500):
|
||||
batch = docs[j:j+500]
|
||||
batch_ids = ids[j:j+500]
|
||||
for retry in range(3):
|
||||
try:
|
||||
collection.add(ids=batch_ids, documents=batch,
|
||||
metadatas=metas[j:j+500])
|
||||
total += len(batch)
|
||||
print(".", end="", flush=True)
|
||||
break
|
||||
except Exception as e:
|
||||
if retry < 2:
|
||||
print(f"R{retry}", end="", flush=True)
|
||||
time.sleep(5)
|
||||
else:
|
||||
print(f"X", end="", flush=True)
|
||||
|
||||
print(f" total={collection.count()}")
|
||||
|
||||
print(f"\n✅ Done. Total: {collection.count()} docs")
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Полная переиндексация ВСЕХ данных в ChromaDB. Работает до победного."""
|
||||
import json, os, sys, time, urllib.request, glob
|
||||
from chromadb import PersistentClient
|
||||
|
||||
EF_URL = "http://localhost:8081/embedding"
|
||||
DB_DIR = os.path.expanduser("~/nubes/chroma_db")
|
||||
|
||||
ALL_DIRS = [
|
||||
"~/nubes/data/elsa_jsonl_wi_en", # 244k WI XML
|
||||
"~/nubes/data/elsa_mdb_jsonl", # 191k rldal
|
||||
"~/nubes/data/elsa_mdb_jsonl/ipsvrap", # 2.5M parts
|
||||
"~/nubes/data/elsa_jsonl_htm_en", # 7k hs2+www
|
||||
"~/nubes/data/elsa_jsonl", # 24k HTM
|
||||
"~/nubes/data/elsa_mdb_jsonl/dbsvrfi", # car ref
|
||||
"~/nubes/data/elsa_mdb_jsonl/dbsvrfz", # PR codes
|
||||
]
|
||||
|
||||
def embed(texts):
|
||||
for attempt in range(5):
|
||||
try:
|
||||
data = json.dumps({"content": texts}).encode()
|
||||
req = urllib.request.Request(EF_URL, data=data,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
body = json.loads(resp.read())
|
||||
if isinstance(body, list):
|
||||
return [item["embedding"][0] for item in body]
|
||||
return body.get("embedding", [])
|
||||
except Exception as e:
|
||||
print(f"\n [embed retry {attempt+1}/5] {e}")
|
||||
time.sleep(15)
|
||||
raise Exception("Embedding failed after 5 retries")
|
||||
|
||||
def main():
|
||||
# Удалить старую БД и создать новую
|
||||
import shutil
|
||||
if os.path.exists(DB_DIR):
|
||||
shutil.rmtree(DB_DIR)
|
||||
print("Старая ChromaDB удалена")
|
||||
|
||||
client = PersistentClient(path=DB_DIR)
|
||||
collection = client.create_collection("elsa_docs")
|
||||
|
||||
# Собираем все JSONL
|
||||
all_files = []
|
||||
for d in ALL_DIRS:
|
||||
path = os.path.expanduser(d)
|
||||
files = sorted(glob.glob(os.path.join(path, "*.jsonl")))
|
||||
all_files.extend(files)
|
||||
print(f"{d}: {len(files)} files")
|
||||
|
||||
print(f"\nВсего {len(all_files)} JSONL файлов")
|
||||
|
||||
total = 0
|
||||
errors = 0
|
||||
|
||||
for fpath in all_files:
|
||||
docs, ids = [], []
|
||||
prefix = os.path.basename(os.path.dirname(fpath)) + "_"
|
||||
|
||||
try:
|
||||
with open(fpath) as fh:
|
||||
for line in fh:
|
||||
try:
|
||||
d = json.loads(line)
|
||||
title = d.get("title", "") or d.get("full_title", "")
|
||||
text = d.get("text", "")
|
||||
doc = (title + "\n" + text).strip()[:1000]
|
||||
if len(doc) < 20:
|
||||
continue
|
||||
docs.append(doc)
|
||||
ids.append(f"{prefix}{os.path.basename(fpath)}_{len(docs)}")
|
||||
except:
|
||||
continue
|
||||
except Exception as e:
|
||||
print(f" [SKIP {os.path.basename(fpath)}] read error: {e}")
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
if not docs:
|
||||
print(f" {os.path.basename(fpath)}: empty")
|
||||
continue
|
||||
|
||||
print(f" {os.path.basename(fpath)}: {len(docs)} docs", end="", flush=True)
|
||||
|
||||
# Бачами по 50
|
||||
for j in range(0, len(docs), 50):
|
||||
batch_docs = docs[j:j+50]
|
||||
batch_ids = ids[j:j+50]
|
||||
|
||||
for retry in range(5):
|
||||
try:
|
||||
embs = embed(batch_docs)
|
||||
collection.add(ids=batch_ids, documents=batch_docs,
|
||||
embeddings=embs, metadatas=[{"source": prefix.strip("_")}] * len(batch_docs))
|
||||
total += len(batch_docs)
|
||||
print(".", end="", flush=True)
|
||||
break
|
||||
except Exception as e:
|
||||
if retry < 4:
|
||||
print(f"R{retry}", end="", flush=True)
|
||||
time.sleep(20)
|
||||
else:
|
||||
print(f"X", end="", flush=True)
|
||||
errors += len(batch_docs)
|
||||
|
||||
time.sleep(0.2)
|
||||
|
||||
print(f" total={collection.count()}")
|
||||
|
||||
print(f"\n✅ Done. Total: {collection.count()} docs. Errors: {errors}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ingest WI JSONL into ChromaDB — one file at a time, batches of 50."""
|
||||
import json, os, sys, time, urllib.request, glob
|
||||
from chromadb import PersistentClient
|
||||
|
||||
EF_URL = "http://localhost:8081/embedding"
|
||||
DB_DIR = os.path.expanduser("~/nubes/chroma_db")
|
||||
# Default: ingest ALL JSONL dirs
|
||||
ALL_DIRS = [
|
||||
os.path.expanduser("~/nubes/data/elsa_jsonl_wi_en"), # 244k WI XML
|
||||
os.path.expanduser("~/nubes/data/elsa_mdb_jsonl"), # 191k rldal
|
||||
os.path.expanduser("~/nubes/data/elsa_mdb_jsonl/ipsvrap"), # 2.5M parts
|
||||
os.path.expanduser("~/nubes/data/elsa_jsonl_htm_en"), # 7k hs2+www
|
||||
os.path.expanduser("~/nubes/data/elsa_jsonl"), # 24k HTM
|
||||
os.path.expanduser("~/nubes/data/elsa_mdb_jsonl/dbsvrfi"), # car ref
|
||||
os.path.expanduser("~/nubes/data/elsa_mdb_jsonl/dbsvrfz"), # PR codes
|
||||
]
|
||||
|
||||
DATA_DIR = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
COLLECTION_NAME = "elsa_docs"
|
||||
|
||||
# Connect to existing ChromaDB
|
||||
client = PersistentClient(path=DB_DIR)
|
||||
try:
|
||||
collection = client.get_or_create_collection(COLLECTION_NAME)
|
||||
except:
|
||||
collection = client.create_collection(COLLECTION_NAME)
|
||||
|
||||
existing = collection.count()
|
||||
print(f"Collection has {existing} docs already")
|
||||
|
||||
# Find remaining files
|
||||
if DATA_DIR:
|
||||
files = sorted(glob.glob(os.path.join(DATA_DIR, "*.jsonl")))
|
||||
else:
|
||||
files = sorted(glob.glob(os.path.join(ALL_DIRS[0], "*.jsonl")))
|
||||
for d in ALL_DIRS[1:]:
|
||||
files.extend(sorted(glob.glob(os.path.join(d, "*.jsonl"))))
|
||||
|
||||
print(f"Processing {len(files)} JSONL files")
|
||||
|
||||
for fpath in files:
|
||||
docs, ids = [], []
|
||||
prefix = os.path.basename(os.path.dirname(fpath)) + "_"
|
||||
with open(fpath) as fh:
|
||||
for line in fh:
|
||||
d = json.loads(line)
|
||||
title = d.get("title", "") or d.get("full_title", "")
|
||||
text = d.get("text", "")
|
||||
doc = (title + "\n" + text).strip()[:1000]
|
||||
if len(doc) < 20:
|
||||
continue
|
||||
docs.append(doc)
|
||||
ids.append(f"{prefix}{os.path.basename(fpath)}_{len(docs)}")
|
||||
|
||||
if not docs:
|
||||
print(f"{os.path.basename(fpath)}: empty, skip")
|
||||
continue
|
||||
|
||||
print(f"{os.path.basename(fpath)}: {len(docs)} docs", end="")
|
||||
|
||||
for j in range(0, len(docs), 50):
|
||||
batch_docs = docs[j:j+50]
|
||||
batch_ids = ids[j:j+50]
|
||||
data = json.dumps({"content": batch_docs}).encode()
|
||||
|
||||
for retry in range(3):
|
||||
try:
|
||||
req = urllib.request.Request(EF_URL, data=data,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
body = json.loads(resp.read())
|
||||
if isinstance(body, list):
|
||||
embs = [item["embedding"][0] for item in body]
|
||||
else:
|
||||
embs = body.get("embedding", [])
|
||||
|
||||
collection.add(ids=batch_ids, documents=batch_docs, embeddings=embs)
|
||||
print(".", end="", flush=True)
|
||||
break
|
||||
except Exception as e:
|
||||
if retry < 2:
|
||||
print(f"R{retry}", end="", flush=True)
|
||||
time.sleep(10)
|
||||
else:
|
||||
print(f"X({e})", end="", flush=True)
|
||||
|
||||
time.sleep(0.3)
|
||||
|
||||
print(f" total={collection.count()}")
|
||||
|
||||
print(f"\nDone. Total: {collection.count()} documents")
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""RAG Query — search ChromaDB + answer via llama.cpp."""
|
||||
import json, os, sys, urllib.request, time
|
||||
import chromadb
|
||||
|
||||
LLAMA_URL = "http://localhost:8080/completion"
|
||||
|
||||
def llama_embed(texts):
|
||||
data = json.dumps({"content": texts}).encode()
|
||||
req = urllib.request.Request("http://localhost:8081/embedding", data=data,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
body = json.loads(resp.read())
|
||||
if isinstance(body, list):
|
||||
return [item["embedding"][0] for item in body]
|
||||
return body.get("embedding", [])
|
||||
|
||||
|
||||
class LlamaEmbedFn:
|
||||
def __call__(self, input):
|
||||
all_embs = []
|
||||
for i in range(0, len(input), 32):
|
||||
batch = input[i:i+32]
|
||||
embs = llama_embed(batch)
|
||||
all_embs.extend(embs if isinstance(embs[0], list) else [embs])
|
||||
return all_embs
|
||||
|
||||
|
||||
def search(collection, query, n=5):
|
||||
results = collection.query(query_texts=[query], n_results=n)
|
||||
docs = []
|
||||
for i in range(len(results["ids"][0])):
|
||||
docs.append({
|
||||
"id": results["ids"][0][i],
|
||||
"doc": results["documents"][0][i],
|
||||
"meta": results["metadatas"][0][i],
|
||||
"distance": results["distances"][0][i],
|
||||
})
|
||||
return docs
|
||||
|
||||
|
||||
def ask_llama(prompt, max_tokens=256):
|
||||
data = json.dumps({
|
||||
"prompt": prompt, "temperature": 0.1,
|
||||
"n_predict": max_tokens,
|
||||
"stop": ["\n\n", "Documentation:", "Question:"],
|
||||
}).encode()
|
||||
req = urllib.request.Request(LLAMA_URL, data=data,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
return json.loads(resp.read()).get("content", "")
|
||||
|
||||
|
||||
def main():
|
||||
db_dir = os.path.expanduser("~/nubes/chroma_db")
|
||||
collection_name = "elsa_docs"
|
||||
|
||||
ef = LlamaEmbedFn()
|
||||
|
||||
client = chromadb.PersistentClient(path=db_dir)
|
||||
collection = client.get_collection(collection_name, embedding_function=ef)
|
||||
|
||||
print(f"Collection '{collection_name}': {collection.count()} documents")
|
||||
|
||||
while True:
|
||||
try:
|
||||
query = input("\n>>> ").strip()
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
print()
|
||||
break
|
||||
|
||||
if not query:
|
||||
continue
|
||||
if query.lower() in ("exit", "quit", "q"):
|
||||
break
|
||||
|
||||
# 1. Поиск
|
||||
results = search(collection, query, n_results=5)
|
||||
|
||||
# 2. Формируем контекст
|
||||
context_parts = []
|
||||
for r in results:
|
||||
title = r["meta"].get("title", "")[:120]
|
||||
ctx = f"[{title}] {r['doc'][:1000]}"
|
||||
context_parts.append(ctx)
|
||||
|
||||
context = "\n\n---\n\n".join(context_parts)
|
||||
|
||||
# 3. Промпт
|
||||
prompt = f"""You are an expert VAG (VW/Audi/Skoda/SEAT) automotive diagnostician.
|
||||
Answer the user's question using ONLY the provided repair documentation below.
|
||||
If the answer is not in the documentation, say "Not found in documentation."
|
||||
Answer in Russian.
|
||||
|
||||
DOCUMENTATION:
|
||||
{context}
|
||||
|
||||
Question: {query}
|
||||
Answer:"""
|
||||
|
||||
print(f"\n[Found {len(results)} matches, sending to LLM...]")
|
||||
answer = ask_llama(prompt)
|
||||
print(f"\n{answer}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""RAG test - embed query, search, answer via Qwen."""
|
||||
import json, os, chromadb, urllib.request
|
||||
|
||||
def embed(texts):
|
||||
data = json.dumps({"content": texts}).encode()
|
||||
req = urllib.request.Request("http://localhost:8081/embedding", data=data,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
body = json.loads(resp.read())
|
||||
if isinstance(body, list):
|
||||
return [item["embedding"][0] for item in body]
|
||||
return body.get("embedding", [])
|
||||
|
||||
os.chdir(os.path.expanduser("~/nubes"))
|
||||
client = chromadb.PersistentClient(path=os.path.expanduser("~/nubes/chroma_db"))
|
||||
collection = client.get_collection("elsa_docs")
|
||||
|
||||
# 1. Embed query
|
||||
query = "What is the tightening torque for fuel filter on VW Phaeton 3.2 VR6 2004?"
|
||||
print(f"Query: {query}")
|
||||
q_emb = embed([query])[0]
|
||||
|
||||
# 2. Search
|
||||
results = collection.query(query_embeddings=[q_emb], n_results=3)
|
||||
print(f"Matches: {len(results['ids'][0])}")
|
||||
for i in range(len(results['ids'][0])):
|
||||
meta = results['metadatas'][0][i] or {}
|
||||
dist = results['distances'][0][i]
|
||||
print(f"\n[{i}] dist={dist:.3f}")
|
||||
print(f" source: {meta.get('source','')}")
|
||||
print(f" title: {meta.get('title','')[:120]}")
|
||||
print(f" text: {results['documents'][0][i][:300]}...")
|
||||
|
||||
# 3. Build prompt for Qwen
|
||||
context = "\n\n".join([f"[{m.get('title','')[:80] if m else ''}]\n{d[:1500]}"
|
||||
for d, m in zip(results['documents'][0], results['metadatas'][0])])
|
||||
|
||||
prompt = f"""You are a VAG automotive diagnostician. Answer using ONLY the documentation below.
|
||||
DOCUMENTATION:
|
||||
{context}
|
||||
|
||||
Question: {query}
|
||||
Answer:"""
|
||||
|
||||
data = json.dumps({"prompt": prompt, "temperature": 0.1, "n_predict": 300,
|
||||
"stop": ["\n\n", "Answer:", "\nThe answer", "\nDocumentation"]}).encode()
|
||||
req = urllib.request.Request("http://localhost:8080/completion", data=data,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=120) as resp:
|
||||
answer = json.loads(resp.read()).get("content", "")
|
||||
|
||||
print(f"\n\n=== LLM Answer ===\n{answer}")
|
||||
|
||||
Reference in New Issue
Block a user