270 lines
8.9 KiB
Bash
Executable File
270 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}"
|
|
PROVIDER_DIR="${ROOT_DIR}/universal_rebuild"
|
|
PROFILE_DIR="${PROFILE_DIR:-}"
|
|
|
|
if [[ "${1:-}" == "--profile" ]]; then
|
|
PROFILE_DIR="${2:-}"
|
|
shift 2
|
|
fi
|
|
|
|
if [[ -n "$PROFILE_DIR" ]]; then
|
|
if [[ ! -d "$PROFILE_DIR" ]]; then
|
|
echo "Error: profile directory not found: $PROFILE_DIR" >&2
|
|
exit 2
|
|
fi
|
|
PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env"
|
|
if [[ -f "$PROFILE_ENV_FILE" ]]; then
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
source "$PROFILE_ENV_FILE"
|
|
set +a
|
|
fi
|
|
fi
|
|
|
|
if [[ -z "$PROFILE_DIR" ]]; then
|
|
echo "Error: --profile <path> is required. Strict mode stores generated files per stand only." >&2
|
|
exit 2
|
|
fi
|
|
|
|
resolve_root_path() {
|
|
local path_value="$1"
|
|
if [[ -z "$path_value" ]]; then
|
|
echo ""
|
|
return
|
|
fi
|
|
if [[ "$path_value" = /* ]]; then
|
|
echo "$path_value"
|
|
return
|
|
fi
|
|
echo "${ROOT_DIR}/${path_value}"
|
|
}
|
|
|
|
SERVICES_FILE_DEFAULT="${ROOT_DIR}/devops/config/services_list.txt"
|
|
if [[ -f "${PROFILE_DIR}/services_list.txt" ]]; then
|
|
SERVICES_FILE_DEFAULT="${PROFILE_DIR}/services_list.txt"
|
|
fi
|
|
SERVICES_FILE="${SERVICES_FILE:-$SERVICES_FILE_DEFAULT}"
|
|
SERVICES_FILE="$(resolve_root_path "$SERVICES_FILE")"
|
|
|
|
# Retry behavior (per service): attempts and delay between attempts.
|
|
REQUEST_DELAY="${REQUEST_DELAY:-0.5}"
|
|
ATTEMPTS="${ATTEMPTS:-3}"
|
|
|
|
# Auth: either an explicit token or a token file (latest *.token if not set).
|
|
TOKEN_FILE="${TOKEN_FILE:-}"
|
|
NUBES_API_TOKEN="${NUBES_API_TOKEN:-}"
|
|
if [[ -n "$TOKEN_FILE" ]]; then
|
|
TOKEN_FILE="$(resolve_root_path "$TOKEN_FILE")"
|
|
fi
|
|
|
|
# Назначение:
|
|
# - Берет список сервисов из services_list.txt.
|
|
# - Для каждого сервиса запрашивает единый spec через API и пишет YAML в
|
|
# ${ROOT_DIR}/universal_rebuild/resources_yaml.
|
|
# - Формат имени файла: ID_имя.yaml (например, 13_s3bucket.yaml).
|
|
#
|
|
# Источники данных:
|
|
# - Список сервисов: ${ROOT_DIR}/devops/config/services_list.txt
|
|
# - API endpoint: ${API_ENDPOINT}
|
|
# - Токен: переменная NUBES_API_TOKEN или файл .token.
|
|
#
|
|
# Формат services_list.txt:
|
|
# - В начале строки: service_id.
|
|
# - Остальное в строке — произвольная заметка для девопса (не используется).
|
|
#
|
|
# Задержки и попытки:
|
|
# - REQUEST_DELAY (секунды) между запросами и ретраями (по умолчанию 0.2).
|
|
# - ATTEMPTS — число попыток на сервис (по умолчанию 3).
|
|
#
|
|
# Логи ошибок:
|
|
# - Список неуспешных сервисов пишется в /tmp/yaml_gen_failures.txt.
|
|
#
|
|
# Токен берется из переменной NUBES_API_TOKEN.
|
|
# Если переменная пуста, берется файл .token из ${ROOT_DIR}:
|
|
# - "последний" = файл с самым новым временем изменения (ls -t | head -n 1).
|
|
if [[ -z "$NUBES_API_TOKEN" ]]; then
|
|
if [[ -z "$TOKEN_FILE" ]]; then
|
|
TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true)
|
|
fi
|
|
if [[ -n "$TOKEN_FILE" && -f "$TOKEN_FILE" ]]; then
|
|
NUBES_API_TOKEN=$(cat "$TOKEN_FILE")
|
|
fi
|
|
fi
|
|
|
|
if [[ -z "$NUBES_API_TOKEN" ]]; then
|
|
echo "Error: NUBES_API_TOKEN or TOKEN_FILE is required" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if [[ ! -f "$SERVICES_FILE" ]]; then
|
|
echo "Error: services file not found: $SERVICES_FILE" >&2
|
|
exit 2
|
|
fi
|
|
|
|
API_ENDPOINT="${NUBES_API_ENDPOINT:-https://deck-api.ngcloud.ru/api/v1/index.cfm}"
|
|
# Auto-detect API style: if endpoint contains "index.cfm" → legacy proxy (?endpoint=),
|
|
# otherwise → new REST gateway (direct paths). No forced /index.cfm normalization.
|
|
|
|
GENERATED_DIR="${PROFILE_DIR}/generated"
|
|
YAML_OUTPUT_DIR_DEFAULT="${GENERATED_DIR}/resources_yaml"
|
|
FAILURES_FILE_DEFAULT="${GENERATED_DIR}/tmp/yaml_gen_failures.txt"
|
|
|
|
YAML_OUTPUT_DIR="${YAML_OUTPUT_DIR:-$YAML_OUTPUT_DIR_DEFAULT}"
|
|
FAILURES_FILE="${FAILURES_FILE:-$FAILURES_FILE_DEFAULT}"
|
|
YAML_OUTPUT_DIR="$(resolve_root_path "$YAML_OUTPUT_DIR")"
|
|
FAILURES_FILE="$(resolve_root_path "$FAILURES_FILE")"
|
|
|
|
# Убедимся, что папка есть. Полной очистки нет: обрабатываем только сервисы из списка.
|
|
mkdir -p "$YAML_OUTPUT_DIR"
|
|
mkdir -p "$(dirname "$FAILURES_FILE")"
|
|
|
|
if [[ ! -f "${YAML_OUTPUT_DIR}/embed.go" ]]; then
|
|
cat > "${YAML_OUTPUT_DIR}/embed.go" <<'EOF'
|
|
package resources_yaml
|
|
|
|
import "embed"
|
|
|
|
// Files embeds unified service YAML specs for provider runtime lookup.
|
|
//
|
|
//go:embed *.yaml
|
|
var Files embed.FS
|
|
EOF
|
|
fi
|
|
|
|
rm -f "$FAILURES_FILE"
|
|
|
|
# Build service_spec_gen binary if missing.
|
|
if [[ ! -x "${PROVIDER_DIR}/bin/service_spec_gen" ]]; then
|
|
echo "Building service_spec_gen..."
|
|
(cd "$PROVIDER_DIR" && go build -o bin/service_spec_gen ./tools/service_spec_gen/)
|
|
fi
|
|
|
|
# Проверка что список сервисов не пустой
|
|
svc_count=$(grep -cv '^\s*$\|^\s*#' "$SERVICES_FILE" || true)
|
|
if [[ "$svc_count" -eq 0 ]]; then
|
|
echo "Error: services list is empty or all commented out: $SERVICES_FILE" >&2
|
|
exit 2
|
|
fi
|
|
echo "Generating YAML for ${svc_count} services from ${SERVICES_FILE}"
|
|
|
|
# Читаем список сервисов, пропуская пустые строки и комментарии.
|
|
while IFS= read -r line; do
|
|
line="${line//$'\r'/}"
|
|
if [[ -z "$line" ]]; then
|
|
continue
|
|
fi
|
|
if [[ "$line" == \#* ]]; then
|
|
continue
|
|
fi
|
|
sid=$(echo "$line" | awk '{print $1}')
|
|
svc_name=""
|
|
|
|
if [[ -z "$sid" ]]; then
|
|
continue
|
|
fi
|
|
|
|
if [[ ! "$sid" =~ ^[0-9]+$ ]]; then
|
|
echo "Error: invalid service id '${sid}' in ${SERVICES_FILE}" >&2
|
|
echo "${sid} invalid_id" >> "$FAILURES_FILE"
|
|
continue
|
|
fi
|
|
|
|
# Если имя не указано в списке — подтягиваем по API.
|
|
if [[ -z "$svc_name" ]]; then
|
|
svc_name=$(python3 - <<PY
|
|
import json, sys, time, urllib.parse, urllib.request
|
|
|
|
sid = "${sid}"
|
|
endpoint = "${API_ENDPOINT}"
|
|
token = "${NUBES_API_TOKEN}"
|
|
max_retries = 3
|
|
|
|
# Auto-detect API style: "index.cfm" → legacy proxy, otherwise → REST gateway.
|
|
if "index.cfm" in endpoint:
|
|
params = urllib.parse.urlencode({"endpoint": f"/services/{sid}"})
|
|
url = f"{endpoint}?{params}"
|
|
else:
|
|
url = f"{endpoint}/services/{sid}"
|
|
|
|
for attempt in range(1, max_retries + 1):
|
|
try:
|
|
req = urllib.request.Request(url)
|
|
if token:
|
|
req.add_header("Authorization", f"Bearer {token}")
|
|
req.add_header("User-Agent", "Mozilla/5.0 (compatible; Terraform-Provider-Nubes/BashGenerator)")
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
data = json.loads(resp.read().decode("utf-8"))
|
|
svc = data.get("svc", {})
|
|
name = svc.get("svcShort") or svc.get("name") or svc.get("title") or f"service_{sid}"
|
|
print(name)
|
|
sys.exit(0)
|
|
except Exception as e:
|
|
if attempt < max_retries:
|
|
time.sleep(2 * attempt)
|
|
else:
|
|
print(f"ERROR: failed to fetch service {sid} after {max_retries} attempts: {e}", file=sys.stderr)
|
|
sys.exit(1)
|
|
PY
|
|
)
|
|
if [[ "$svc_name" == ERROR:* ]]; then
|
|
echo "$svc_name" >&2
|
|
echo "${sid} api_error" >> "$FAILURES_FILE"
|
|
continue
|
|
fi
|
|
fi
|
|
|
|
# Один YAML на сервис формируется Go-генератором.
|
|
echo "Generating unified spec for ${sid} (${svc_name})"
|
|
success=0
|
|
# Generator normalizes names, so verify output by ID prefix only.
|
|
output_glob="${YAML_OUTPUT_DIR}/${sid}_*.yaml"
|
|
|
|
# Для выбранного сервиса удаляем старый YAML и генерируем заново с ретраями.
|
|
rm -f "$output_glob"
|
|
|
|
# Генерация YAML для выбранного сервиса с ретраями.
|
|
for attempt in $(seq 1 "$ATTEMPTS"); do
|
|
(
|
|
cd "$PROVIDER_DIR"
|
|
NUBES_API_TOKEN="$NUBES_API_TOKEN" \
|
|
NUBES_SERVICE_ID="$sid" \
|
|
NUBES_SERVICE_NAME="$svc_name" \
|
|
NUBES_API_ENDPOINT="$API_ENDPOINT" \
|
|
NUBES_OUTPUT_DIR="$YAML_OUTPUT_DIR" \
|
|
"${PROVIDER_DIR}/bin/service_spec_gen"
|
|
) && success=1 || success=0
|
|
|
|
if [[ "$success" -eq 1 ]]; then
|
|
# Success means output file exists for the service ID.
|
|
if ! compgen -G "$output_glob" > /dev/null; then
|
|
echo "Error: output not created ${output_glob} for ${sid} (${svc_name})" >&2
|
|
success=0
|
|
fi
|
|
fi
|
|
|
|
if [[ "$success" -eq 1 ]]; then
|
|
break
|
|
fi
|
|
echo "Retry ${attempt}/${ATTEMPTS} failed for ${sid} (${svc_name})" >&2
|
|
sleep "$REQUEST_DELAY"
|
|
done
|
|
|
|
# Фиксируем неудачные сервисы для анализа.
|
|
if [[ "$success" -ne 1 ]]; then
|
|
echo "${sid} ${svc_name}" >> "$FAILURES_FILE"
|
|
fi
|
|
|
|
sleep "$REQUEST_DELAY"
|
|
|
|
done < "$SERVICES_FILE"
|
|
|
|
echo "Done. YAML files are in ${YAML_OUTPUT_DIR}"
|
|
if [[ -f "$FAILURES_FILE" ]]; then
|
|
echo "Failed services (see $FAILURES_FILE):" >&2
|
|
cat "$FAILURES_FILE" >&2
|
|
exit 1
|
|
fi |