feat: скрипт сравнения трёх LLM + сессия 29 мая

2026-05-29 08:48:30 +03:00
2 changed files with 254 additions and 0 deletions
@@ -0,0 +1,51 @@
+# Сессия 2026-05-29 — Сравнение LLM, модели DeepSeek, модели aillm.ru
+
+## Хронология
+
+### 10:00 — Вопрос о модели LLM
+- Используем `gpt-oss-120b` (120B open-source) на `api.aillm.ru/v1`
+- Изначально задумано как DeepSeek, но провайдер — aillm.ru
+
+### 10:15 — DeepSeek V4: модели и цены
+**DeepSeek V4 Flash (`deepseek-v4-flash`):**
+- Кеш hit: $0.0028 / 1M токенов
+- Кеш miss: $0.14 / 1M
+- Выход: $0.28 / 1M
+- 1M контекст, 384K max output, 2500 concurrent
+
+**DeepSeek V4 Pro (`deepseek-v4-pro`):**
+- До 31.05 акция 75% off
+- С 01.06 официальная цена = ¼ original (т.е. та же, что со скидкой)
+- Кеш hit: $0.0145 / 1M
+- Вход: $1.74 → фикс $0.435 / 1M
+- Выход: $3.48 → фикс $0.87 / 1M
+- 500 concurrent
+
+### 10:30 — Создан `tools/compare_llm.py`
+Скрипт сравнения трёх моделей на одинаковом промпте (VIN VW, DTC P0301+P0171+P0420, параметры ЭБУ).
+
+### 10:45 — Запуск сравнения
+
+| Модель | Время | Вход | Выход | Стоимость |
+|--------|-------|------|-------|-----------|
+| DeepSeek V4 Flash | 56.9с | 427 | 6,494 | $0.0019 |
+| DeepSeek V4 Pro | 223.5с | 427 | 10,323 | $0.0030 |
+| Aillm.ru (gpt-oss-120b) | 103.3с | 492 | 7,003 | ? |
+
+- V4 Flash: быстрый, конкретика по VW, лучший price/quality
+- V4 Pro: самый глубокий ответ (думал 223с — режим thinking)
+- Aillm 120b: адекватно, структурированно, но медленнее Flash
+
+**Вывод:** 120b юзабелен для прототипа, DeepSeek V4 Flash — оптимальная замена при переходе в прод.
+
+### 11:00 — Модели aillm.ru
+Запросом к `/v1/models` получен список:
+- `gpt-oss-120b` ← текущая
+- `qwen3-6-27b-fp8` (27B)
+- `qwen3.5-9b` (9B)
+- `whisper-large-v3-turbo` (речь)
+
+Для диагноза альтернатив 120b нет.
+
+### 11:15 — Dify-совместимость
+aillm.ru отдаёт OpenAI-совместимый API → Dify подключается как кастомный провайдер.
@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""Compare three LLMs: DeepSeek V4 Flash, DeepSeek V4 Pro, Aillm.ru.
+
+Usage:
+  DEEPSEEK_API_KEY=sk-... python tools/compare_llm.py
+"""
+
+import os
+import sys
+import time
+from pathlib import Path
+
+import requests
+import yaml
+
+TEST_DATA = {
+    "vin": "WVWZZZ1KZAW123456",
+    "dtc_codes": [
+        {"code": "P0301", "status": "stored", "description": "Misfire cylinder 1"},
+        {"code": "P0171", "status": "stored", "description": "System too lean bank 1"},
+        {"code": "P0420", "status": "pending", "description": "Catalyst efficiency low bank 1"},
+    ],
+    "parameters": [
+        {"pid_code": "0105", "name": "coolant_temp", "value": 87.0, "unit": "C"},
+        {"pid_code": "010C", "name": "rpm", "value": 680, "unit": "rpm"},
+        {"pid_code": "010D", "name": "speed", "value": 0, "unit": "km/h"},
+        {"pid_code": "010B", "name": "map", "value": 35, "unit": "kPa"},
+        {"pid_code": "010F", "name": "iat", "value": 31, "unit": "C"},
+        {"pid_code": "0104", "name": "engine_load", "value": 18.0, "unit": "%"},
+        {"pid_code": "0106", "name": "stft_b1", "value": -8.6, "unit": "%"},
+        {"pid_code": "0107", "name": "ltft_b1", "value": -12.5, "unit": "%"},
+        {"pid_code": "0111", "name": "throttle_pos", "value": 12, "unit": "%"},
+    ],
+}
+
+SYSTEM_PROMPT = (
+    "You are an auto diagnostic expert with 20 years of experience. "
+    "Analyze OBD2 error codes and ECU parameters. Give a DEEP, DETAILED analysis.\n\n"
+    "RULES:\n"
+    "1. Do NOT limit yourself to a brief summary - give FULL analysis of each error and parameter.\n"
+    "2. For each error: explain what it means, ALL possible causes (common to rare), "
+    "which parameters confirm/refute each version.\n"
+    "3. Analyze RELATIONSHIPS between errors and parameters.\n"
+    "4. Give confidence percentages for EACH conclusion.\n"
+    "5. If data is insufficient - list SPECIFIC PIDs to read additionally, explain why.\n"
+    "6. Suggest action plan: what to check FIRST (most likely and cheapest), THEN what.\n"
+    "7. For each action: HOW to check, WHAT to look for, normal/deviant values.\n"
+    "8. Add 'If not helped' section - plan B for each item.\n"
+    "9. NEVER give categorical commands 'replace part X' without 100% confidence. "
+    "Write 'check X before replacing Y'.\n"
+    "10. Write in Russian, accessible but TECHNICALLY PRECISE. Use tables where appropriate.\n\n"
+    "OUTPUT FORMAT:\n"
+    "## Diagnosis (detailed)\n...\n## Error analysis\n...\n## Parameter analysis\n...\n"
+    "## Relationships\n...\n## Action plan (by priority)\n...\n"
+    "## What data is missing\n...\n## Confidence\n..."
+)
+
+
+def build_user_prompt(data: dict) -> str:
+    lines = [f"**VIN:** {data['vin']}", ""]
+
+    if data["dtc_codes"]:
+        lines.append("**Error codes:**")
+        for d in data["dtc_codes"]:
+            lines.append(f"- {d['code']} ({d['status']}): {d.get('description', '')}")
+        lines.append("")
+
+    if data["parameters"]:
+        lines.append("**ECU parameters:**")
+        for p in data["parameters"]:
+            lines.append(f"- {p['name']}: {p['value']} {p['unit']}")
+        lines.append("")
+
+    lines.append("Conduct a full diagnosis.")
+    return "\n".join(lines)
+
+
+def call_llm(api_key: str, base_url: str, model: str, messages: list[dict]) -> dict:
+    start = time.time()
+    resp = requests.post(
+        f"{base_url.rstrip('/')}/chat/completions",
+        headers={
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        },
+        json={
+            "model": model,
+            "messages": messages,
+            "temperature": 0.3,
+            "max_tokens": 32000,
+        },
+        timeout=300,
+    )
+    elapsed = time.time() - start
+    resp.raise_for_status()
+    data = resp.json()
+    usage = data.get("usage", {})
+    return {
+        "content": data["choices"][0]["message"]["content"],
+        "elapsed": elapsed,
+        "prompt_tokens": usage.get("prompt_tokens", "?"),
+        "completion_tokens": usage.get("completion_tokens", "?"),
+        "total_tokens": usage.get("total_tokens", "?"),
+    }
+
+
+def print_answer(label: str, result: dict | None):
+    if result is None:
+        return
+    print()
+    border = "=" * 20
+    print(f"{border} {label} {border}")
+    print()
+    content = result["content"]
+    print(content)
+    print()
+
+
+def main():
+    config_path = Path(__file__).parent.parent / "config.yaml"
+    with open(config_path) as f:
+        cfg = yaml.safe_load(f)
+
+    aillm_key = cfg["llm"]["api_key"]
+    aillm_model = cfg["llm"].get("model", "gpt-oss-120b")
+    aillm_url = cfg["llm"].get("base_url", "https://api.aillm.ru/v1")
+
+    ds_key = os.environ.get("DEEPSEEK_API_KEY")
+    if not ds_key:
+        print("ERROR: DEEPSEEK_API_KEY not set")
+        sys.exit(1)
+
+    ds_url = "https://api.deepseek.com/v1"
+
+    user_text = build_user_prompt(TEST_DATA)
+    messages = [
+        {"role": "system", "content": SYSTEM_PROMPT},
+        {"role": "user", "content": user_text},
+    ]
+
+    # Print the prompt being sent
+    print("=" * 70)
+    print("PROMPT BEING SENT")
+    print("=" * 70)
+    print()
+    print("--- SYSTEM ---")
+    print(SYSTEM_PROMPT[:300] + "...")
+    print()
+    print("--- USER ---")
+    print(user_text)
+    print()
+
+    models = [
+        ("DeepSeek V4 Flash", "deepseek-v4-flash", ds_key, ds_url),
+        ("DeepSeek V4 Pro", "deepseek-v4-pro", ds_key, ds_url),
+        ("Aillm.ru", aillm_model, aillm_key, aillm_url),
+    ]
+
+    results = {}
+    for label, model, key, url in models:
+        print("-" * 70)
+        print(f"  Calling {label} ({model})...", end=" ", flush=True)
+        try:
+            r = call_llm(key, url, model, messages)
+            results[label] = r
+            print(f"DONE  {r['elapsed']:.1f}s  "
+                  f"in={r['prompt_tokens']} out={r['completion_tokens']}")
+        except Exception as e:
+            print(f"ERROR: {e}")
+            results[label] = None
+
+    print()
+    print("=" * 70)
+    print()
+
+    for label in [m[0] for m in models]:
+        print_answer(label, results[label])
+
+    # Summary table
+    print("=" * 70)
+    print("SUMMARY")
+    print("=" * 70)
+    print()
+    print(f"{'Model':<25} {'Time':>8} {'Input':>8} {'Output':>8} {'Cost':>12}")
+    print("-" * 65)
+    for label, _, key, url in models:
+        r = results[label]
+        if r is None:
+            print(f"{label:<25} {'FAIL':>8}")
+        else:
+            if "DeepSeek" in label:
+                cost = (r["prompt_tokens"] / 1_000_000 * 0.14 +
+                        r["completion_tokens"] / 1_000_000 * 0.28)
+                cost_str = f"${cost:.6f}"
+            else:
+                cost_str = "?"
+            print(f"{label:<25} {r['elapsed']:>7.1f}s {r['prompt_tokens']:>8} "
+                  f"{r['completion_tokens']:>8} {cost_str:>12}")
+    print()
+
+
+if __name__ == "__main__":
+    main()