tf_provider/devops/01_generate_yamls.sh

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}"
PROVIDER_DIR="${ROOT_DIR}/provider"
PROFILE_DIR="${PROFILE_DIR:-}"

if [[ "${1:-}" == "--profile" ]]; then
  PROFILE_DIR="${2:-}"
  shift 2
fi

if [[ -n "$PROFILE_DIR" ]]; then
  if [[ ! -d "$PROFILE_DIR" ]]; then
    echo "Error: profile directory not found: $PROFILE_DIR" >&2
    exit 2
  fi
  PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env"
  if [[ -f "$PROFILE_ENV_FILE" ]]; then
    set -a
    # shellcheck disable=SC1090
    source "$PROFILE_ENV_FILE"
    set +a
  fi
fi

if [[ -z "$PROFILE_DIR" ]]; then
  echo "Error: --profile <path> is required. Strict mode stores generated files per stand only." >&2
  exit 2
fi

resolve_root_path() {
  local path_value="$1"
  if [[ -z "$path_value" ]]; then
    echo ""
    return
  fi
  if [[ "$path_value" = /* ]]; then
    echo "$path_value"
    return
  fi
  echo "${ROOT_DIR}/${path_value}"
}

SERVICES_FILE_DEFAULT="${ROOT_DIR}/devops/config/services_list.txt"
if [[ -f "${PROFILE_DIR}/services_list.txt" ]]; then
  SERVICES_FILE_DEFAULT="${PROFILE_DIR}/services_list.txt"
fi
SERVICES_FILE="${SERVICES_FILE:-$SERVICES_FILE_DEFAULT}"
SERVICES_FILE="$(resolve_root_path "$SERVICES_FILE")"

# Retry behavior (per service): attempts and delay between attempts.
REQUEST_DELAY="${REQUEST_DELAY:-0.5}"
ATTEMPTS="${ATTEMPTS:-3}"

# Auth: either an explicit token or a token file (latest *.token if not set).
TOKEN_FILE="${TOKEN_FILE:-}"
NUBES_API_TOKEN="${NUBES_API_TOKEN:-}"
if [[ -n "$TOKEN_FILE" ]]; then
  TOKEN_FILE="$(resolve_root_path "$TOKEN_FILE")"
fi

# Назначение:
# - Берет список сервисов из services_list.txt.
# - Для каждого сервиса запрашивает единый spec через API и пишет YAML в
#   ${ROOT_DIR}/provider/resources_yaml.
# - Формат имени файла: ID_имя.yaml (например, 13_s3bucket.yaml).
#
# Источники данных:
# - Список сервисов: ${ROOT_DIR}/devops/config/services_list.txt
# - API endpoint: ${API_ENDPOINT}
# - Токен: переменная NUBES_API_TOKEN или файл .token.
#
# Формат services_list.txt:
# - В начале строки: service_id.
# - Остальное в строке — произвольная заметка для девопса (не используется).
#
# Задержки и попытки:
# - REQUEST_DELAY (секунды) между запросами и ретраями (по умолчанию 0.2).
# - ATTEMPTS — число попыток на сервис (по умолчанию 3).
#
# Логи ошибок:
# - Список неуспешных сервисов пишется в /tmp/yaml_gen_failures.txt.
#
# Токен берется из переменной NUBES_API_TOKEN.
# Если переменная пуста, берется файл .token из ${ROOT_DIR}:
# - "последний" = файл с самым новым временем изменения (ls -t | head -n 1).
if [[ -z "$NUBES_API_TOKEN" ]]; then
  if [[ -z "$TOKEN_FILE" ]]; then
    TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true)
  fi
  if [[ -n "$TOKEN_FILE" && -f "$TOKEN_FILE" ]]; then
    NUBES_API_TOKEN=$(cat "$TOKEN_FILE")
  fi
fi

if [[ -z "$NUBES_API_TOKEN" ]]; then
  echo "Error: NUBES_API_TOKEN or TOKEN_FILE is required" >&2
  exit 2
fi

if [[ ! -f "$SERVICES_FILE" ]]; then
  echo "Error: services file not found: $SERVICES_FILE" >&2
  exit 2
fi

API_ENDPOINT="${NUBES_API_ENDPOINT:-https://deck-api.ngcloud.ru/api/v1/index.cfm}"
# Auto-detect API style: if endpoint contains "index.cfm" → legacy proxy (?endpoint=),
# otherwise → new REST gateway (direct paths). No forced /index.cfm normalization.

GENERATED_DIR="${PROFILE_DIR}/generated"
YAML_OUTPUT_DIR_DEFAULT="${GENERATED_DIR}/resources_yaml"
FAILURES_FILE_DEFAULT="${GENERATED_DIR}/tmp/yaml_gen_failures.txt"

YAML_OUTPUT_DIR="${YAML_OUTPUT_DIR:-$YAML_OUTPUT_DIR_DEFAULT}"
FAILURES_FILE="${FAILURES_FILE:-$FAILURES_FILE_DEFAULT}"
YAML_OUTPUT_DIR="$(resolve_root_path "$YAML_OUTPUT_DIR")"
FAILURES_FILE="$(resolve_root_path "$FAILURES_FILE")"

# Убедимся, что папка есть. Полной очистки нет: обрабатываем только сервисы из списка.
mkdir -p "$YAML_OUTPUT_DIR"
mkdir -p "$(dirname "$FAILURES_FILE")"

if [[ ! -f "${YAML_OUTPUT_DIR}/embed.go" ]]; then
  cat > "${YAML_OUTPUT_DIR}/embed.go" <<'EOF'
package resources_yaml

import "embed"

// Files embeds unified service YAML specs for provider runtime lookup.
//
//go:embed *.yaml
var Files embed.FS
EOF
fi

rm -f "$FAILURES_FILE"

# Build service_spec_gen binary if missing.
if [[ ! -x "${ROOT_DIR}/TOOLS/bin/yaml-generator" ]]; then
  echo "Building service_spec_gen..."
  # Binary pre-built in TOOLS/bin/yaml-generator
fi

# Проверка что список сервисов не пустой
svc_count=$(grep -cv '^\s*$\|^\s*#' "$SERVICES_FILE" || true)
if [[ "$svc_count" -eq 0 ]]; then
  echo "Error: services list is empty or all commented out: $SERVICES_FILE" >&2
  exit 2
fi
echo "Generating YAML for ${svc_count} services from ${SERVICES_FILE}"

# Читаем список сервисов, пропуская пустые строки и комментарии.
while IFS= read -r line; do
  line="${line//$'\r'/}"
  if [[ -z "$line" ]]; then
    continue
  fi
  if [[ "$line" == \#* ]]; then
    continue
  fi
  sid=$(echo "$line" | awk '{print $1}')
  svc_name=""

  if [[ -z "$sid" ]]; then
    continue
  fi

  if [[ ! "$sid" =~ ^[0-9]+$ ]]; then
    echo "Error: invalid service id '${sid}' in ${SERVICES_FILE}" >&2
    echo "${sid} invalid_id" >> "$FAILURES_FILE"
    continue
  fi

  # Если имя не указано в списке — подтягиваем по API.
  if [[ -z "$svc_name" ]]; then
    svc_name=$(python3 - <<PY
import json, sys, time, urllib.parse, urllib.request

sid = "${sid}"
endpoint = "${API_ENDPOINT}"
token = "${NUBES_API_TOKEN}"
max_retries = 3

# Auto-detect API style: "index.cfm" → legacy proxy, otherwise → REST gateway.
if "index.cfm" in endpoint:
    params = urllib.parse.urlencode({"endpoint": f"/services/{sid}"})
    url = f"{endpoint}?{params}"
else:
    url = f"{endpoint}/services/{sid}"

for attempt in range(1, max_retries + 1):
    try:
        req = urllib.request.Request(url)
        if token:
            req.add_header("Authorization", f"Bearer {token}")
        req.add_header("User-Agent", "Mozilla/5.0 (compatible; Terraform-Provider-Nubes/BashGenerator)")
        with urllib.request.urlopen(req, timeout=30) as resp:
            data = json.loads(resp.read().decode("utf-8"))
        svc = data.get("svc", {})
        name = svc.get("svcShort") or svc.get("name") or svc.get("title") or f"service_{sid}"
        print(name)
        sys.exit(0)
    except Exception as e:
        if attempt < max_retries:
            time.sleep(2 * attempt)
        else:
            print(f"ERROR: failed to fetch service {sid} after {max_retries} attempts: {e}", file=sys.stderr)
            sys.exit(1)
PY
)
    if [[ "$svc_name" == ERROR:* ]]; then
      echo "$svc_name" >&2
      echo "${sid} api_error" >> "$FAILURES_FILE"
      continue
    fi
  fi

  # Один YAML на сервис формируется Go-генератором.
  echo "Generating unified spec for ${sid} (${svc_name})"
  success=0
  # Generator normalizes names, so verify output by ID prefix only.
  output_glob="${YAML_OUTPUT_DIR}/${sid}_*.yaml"

  # Для выбранного сервиса удаляем старый YAML и генерируем заново с ретраями.
  rm -f "$output_glob"

  # Генерация YAML для выбранного сервиса с ретраями.
  for attempt in $(seq 1 "$ATTEMPTS"); do
    (
      cd "$PROVIDER_DIR"
      NUBES_API_TOKEN="$NUBES_API_TOKEN" \
      NUBES_SERVICE_ID="$sid" \
      NUBES_SERVICE_NAME="$svc_name" \
      NUBES_API_ENDPOINT="$API_ENDPOINT" \
      NUBES_OUTPUT_DIR="$YAML_OUTPUT_DIR" \
      "${ROOT_DIR}/TOOLS/bin/yaml-generator"
    ) && success=1 || success=0

    if [[ "$success" -eq 1 ]]; then
      # Success means output file exists for the service ID.
      if ! compgen -G "$output_glob" > /dev/null; then
        echo "Error: output not created ${output_glob} for ${sid} (${svc_name})" >&2
        success=0
      fi
    fi

    if [[ "$success" -eq 1 ]]; then
      break
    fi
    echo "Retry ${attempt}/${ATTEMPTS} failed for ${sid} (${svc_name})" >&2
    sleep "$REQUEST_DELAY"
  done

  # Фиксируем неудачные сервисы для анализа.
  if [[ "$success" -ne 1 ]]; then
    echo "${sid} ${svc_name}" >> "$FAILURES_FILE"
  fi

  sleep "$REQUEST_DELAY"

done < "$SERVICES_FILE"

echo "Done. YAML files are in ${YAML_OUTPUT_DIR}"
if [[ -f "$FAILURES_FILE" ]]; then
  echo "Failed services (see $FAILURES_FILE):" >&2
  cat "$FAILURES_FILE" >&2
  exit 1
fi