fission-console/test_llm_syntax.sh

#!/usr/bin/env bash
# Полное тестирование LLM syntax-check endpoint
# POST /console/api/ai/check  {language, code} -> {ok, result}

API="https://fission.kube5s.ru/console/api/ai/check"
AUTH='-H "X-Test-Sub: livetest@test.local"'
CT='-H "Content-Type: application/json"'

PASS=0
FAIL=0
HALLUCINATION=0

check() {
  local id="$1"
  local desc="$2"
  local lang="$3"
  local expect="$4"   # "ok" or "fail"
  local code="$5"

  # Escape code for JSON
  local escaped
  escaped=$(printf '%s' "$code" | python3 -c "import sys,json; print(json.dumps(sys.stdin.read()))")

  local body="{\"language\":\"${lang}\",\"code\":${escaped}}"

  local resp
  resp=$(curl -s -X POST "$API" \
    -H "X-Test-Sub: livetest@test.local" \
    -H "Content-Type: application/json" \
    -d "$body" 2>/dev/null)

  local ok_val
  ok_val=$(echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('ok','?'))" 2>/dev/null)
  local result_val
  result_val=$(echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('result','')[:120])" 2>/dev/null)

  local status
  if [[ "$expect" == "ok" && "$ok_val" == "True" ]]; then
    status="PASS"
    ((PASS++))
  elif [[ "$expect" == "fail" && "$ok_val" == "False" ]]; then
    status="PASS"
    ((PASS++))
  else
    status="FAIL"
    ((FAIL++))
  fi

  echo "[$status] $id | $lang | expect=$expect got=$ok_val | $desc"
  if [[ "$status" == "FAIL" ]]; then
    echo "       resp: $result_val"
  fi
}

hallucination_check() {
  local id="$1"
  local desc="$2"
  local lang="$3"
  local code="$4"
  local must_not_contain="$5"  # keyword in result that would be hallucination

  local escaped
  escaped=$(printf '%s' "$code" | python3 -c "import sys,json; print(json.dumps(sys.stdin.read()))")

  local body="{\"language\":\"${lang}\",\"code\":${escaped}}"
  local resp
  resp=$(curl -s -X POST "$API" \
    -H "X-Test-Sub: livetest@test.local" \
    -H "Content-Type: application/json" \
    -d "$body" 2>/dev/null)

  local result_val
  result_val=$(echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('result',''))" 2>/dev/null)
  local ok_val
  ok_val=$(echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('ok','?'))" 2>/dev/null)

  if echo "$result_val" | grep -qi "$must_not_contain"; then
    echo "[HALLUCINATION] $id | $lang | $desc"
    echo "       Found '$must_not_contain' in: $(echo "$result_val" | head -c 200)"
    ((HALLUCINATION++))
  else
    echo "[HALL-OK] $id | $lang | $desc | ok=$ok_val"
    ((PASS++))
  fi
}

echo "========================================"
echo " LLM SYNTAX CHECK — FULL TEST SUITE"
echo "========================================"
echo ""

echo "--- NODEJS ---"

# CORRECT
check N01 "правильный модуль с handler" nodejs ok \
'module.exports = async function(context) {
  return { status: 200, body: JSON.stringify({ ok: true }) };
};'

# CORRECT: named export handler
check N02 "named export handler" nodejs ok \
'async function handler(context) {
  const name = context.request.query.name || "World";
  return { status: 200, body: "Hello " + name };
}
module.exports = { handler };'

# SYNTAX ERROR: missing closing brace
check N03 "синтаксическая ошибка - нет закрывающей скобки" nodejs fail \
'module.exports = async function(context) {
  return { status: 200, body: "hello" };
// забыли закрыть функцию'

# SYNTAX ERROR: invalid JS
check N04 "невалидный JS" nodejs fail \
'const x = @@@;
module.exports = function(ctx) { return x; };'

# WRONG ENTRYPOINT: exports nothing
check N05 "нет module.exports (нет entrypoint)" nodejs fail \
'async function doStuff(context) {
  return { status: 200, body: "ok" };
}'

# SEMANTIC: returns wrong shape (no status)
check N06 "возвращает строку вместо объекта" nodejs fail \
'module.exports = async function(context) {
  return "just a string";
};'

echo ""
echo "--- PYTHON ---"

# CORRECT
check P01 "правильный python handler" python ok \
'def main(data):
    name = data.get("name", "World")
    return "Hello " + name'

# CORRECT: with context signature
check P02 "handler с двумя аргументами" python ok \
'def main(context, event):
    return {"status": 200, "body": "ok"}'

# SYNTAX ERROR: IndentationError
check P03 "IndentationError" python fail \
'def main(data):
return "hello"'

# SYNTAX ERROR: invalid token
check P04 "невалидный синтаксис Python" python fail \
'def main(data):
    x = @@@
    return x'

# WRONG ENTRYPOINT
check P05 "нет функции main/handler" python fail \
'def process(data):
    return "ok"'

# SEMANTIC: unused import, no return
check P06 "функция без return" python fail \
'def main(data):
    name = data.get("name", "World")
    print("Hello " + name)'

echo ""
echo "--- GO ---"

# CORRECT
check G01 "правильный Go handler" go ok \
'package main

import (
    "net/http"
    "fmt"
)

func Handler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hello World")
}'

# SYNTAX ERROR: missing import close
check G02 "синтаксическая ошибка Go" go fail \
'package main

import (
    "net/http"
    "fmt"

func Handler(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hello")
}'

# WRONG SIGNATURE
check G03 "неверная сигнатура Handler" go fail \
'package main

import "fmt"

func Handler(name string) string {
    return fmt.Sprintf("Hello %s", name)
}'

# WRONG PACKAGE
check G04 "неверный package" go fail \
'package utils

import "net/http"

func Handler(w http.ResponseWriter, r *http.Request) {}'

echo ""
echo "--- RUBY ---"

# CORRECT
check R01 "правильный Ruby handler" ruby ok \
'def handler(context)
  { status: 200, body: "Hello World" }
end'

# SYNTAX ERROR
check R02 "синтаксическая ошибка Ruby" ruby fail \
'def handler(context)
  { status: 200 body: "missing comma" }
end'

# NO ENTRYPOINT
check R03 "нет функции handler" ruby fail \
'def process(ctx)
  { status: 200, body: "ok" }
end'

echo ""
echo "--- PHP ---"

# CORRECT
check H01 "правильный PHP handler" php ok \
'<?php
function handler(array $context): array {
    $name = $context["name"] ?? "World";
    return ["status" => 200, "body" => "Hello " . $name];
}'

# SYNTAX ERROR: missing semicolon
check H02 "синтаксическая ошибка PHP" php fail \
'<?php
function handler(array $context): array {
    $name = $context["name"] ?? "World"
    return ["status" => 200, "body" => "Hello " . $name];
}'

# NO ENTRYPOINT
check H03 "нет функции handler в PHP" php fail \
'<?php
function process(array $data): array {
    return ["status" => 200];
}'

echo ""
echo "--- PERL ---"

# CORRECT
check L01 "правильный Perl handler" perl ok \
'sub handler {
    my ($env) = @_;
    return [200, ["Content-Type" => "text/plain"], ["Hello World"]];
}'

# SYNTAX ERROR
check L02 "синтаксическая ошибка Perl" perl fail \
'sub handler {
    my $env = @_
    return [200, [], ["ok"]];
}'

# NO ENTRYPOINT
check L03 "нет sub handler в Perl" perl fail \
'sub process {
    my ($env) = @_;
    return [200, [], ["ok"]];
}'

echo ""
echo "--- ГРАНИЧНЫЕ СЛУЧАИ ---"

# Empty code
check E01 "пустой код" nodejs fail ""

# Gibberish
check E02 "gibberish" python fail \
'asdfjklasdfjkl %%% @@@ ???'

# Code from another language in wrong field (Python code as nodejs)
check E03 "Python код помечен как nodejs" nodejs fail \
'def main(data):
    return "wrong lang"'

# SQL injection in code field - should not crash
check E04 "SQL injection в коде" python fail \
"'; DROP TABLE functions; --"

# Very minimal but correct
check E05 "минимально корректный nodejs" nodejs ok \
'module.exports = (ctx) => ({ status: 200, body: "ok" });'

echo ""
echo "--- HALLUCINATION CHECKS ---"
echo "(проверяем что LLM не выдумывает несуществующие ошибки)"

# Perfect Node.js — LLM не должен говорить об ошибках синтаксиса
hallucination_check H_N01 "идеальный nodejs — нет синтаксических ошибок" nodejs \
'module.exports = async function(context) {
  const x = 42;
  const y = x * 2;
  return { status: 200, body: String(y) };
};' "синтаксическ"

# Perfect Python — не должно быть "ошибок"
hallucination_check H_P01 "идеальный python — нет ошибок" python \
'def main(context, event):
    result = {"message": "hello", "count": 42}
    return result' "ошибка"

# Perfect Go
hallucination_check H_G01 "идеальный go — нет ошибок" go \
'package main

import (
    "encoding/json"
    "net/http"
)

func Handler(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Content-Type", "application/json")
    json.NewEncoder(w).Encode(map[string]string{"ok": "true"})
}' "синтаксическ"

echo ""
echo "========================================"
echo " РЕЗУЛЬТАТЫ"
echo "========================================"
echo "  PASS:           $PASS"
echo "  FAIL:           $FAIL"
echo "  HALLUCINATIONS: $HALLUCINATION"
echo "========================================"