refactor: restructure project — separate tools, config, generated, provider
TOOLS/ — generators + scripts + config + ARCHITECTURE.md ├── yaml-generator/ (code) ├── resource-generator/ (code) ├── docs-generator/ (code) ├── scripts/ (← devops/*.sh) ├── config/ (← devops/profiles/ + devops/config/) │ ├── test/ profile.env, services_list.txt, operation_timeouts.json │ ├── prod/ │ └── dev/ └── ARCHITECTURE.md (← devops/ARCHITECTURE.md) generated/ — pipeline output only (gitignored) ├── test/resources_yaml/, go/, docs/, provider_build/ ├── prod/ └── dev/ provider/ — code only, no generated files resources_yaml/ — DELETED (generated) internal/resources_gen/ — DELETED (generated) devops/ — removed (replaced by TOOLS/scripts + TOOLS/config + generated/) Generators now fail if NUBES_*_DIR not set (no defaults to provider/). Provider requires pipeline to populate resources_gen/ before build.
This commit is contained in:
“Naeel”
Executable
+230
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DEFAULT_BASE_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
BASE_DIR="${BASE_DIR:-${DEFAULT_BASE_DIR}}"
|
||||
CLIENT_ID="${CLIENT_ID:-account-console}"
|
||||
TOKEN_URL="${TOKEN_URL:-https://keycloak.nubes.ru/realms/cloud/protocol/openid-connect/token}"
|
||||
THRESHOLD_MIN="${THRESHOLD_MIN:-10}"
|
||||
FORCE_REFRESH="${FORCE_REFRESH:-0}"
|
||||
|
||||
usage() {
|
||||
echo "Usage:" >&2
|
||||
echo " $0 save [json_file|-]" >&2
|
||||
echo " $0 refresh" >&2
|
||||
echo " $0 current" >&2
|
||||
echo "" >&2
|
||||
echo "Env vars:" >&2
|
||||
echo " BASE_DIR=${DEFAULT_BASE_DIR}" >&2
|
||||
echo " THRESHOLD_MIN=10" >&2
|
||||
echo " FORCE_REFRESH=1" >&2
|
||||
echo " TOKEN_URL=https://keycloak.nubes.ru/realms/cloud/protocol/openid-connect/token" >&2
|
||||
echo " CLIENT_ID=account-console" >&2
|
||||
}
|
||||
|
||||
cmd="${1:-}"
|
||||
|
||||
case "$cmd" in
|
||||
save)
|
||||
json_source="${2:--}"
|
||||
if [[ "$json_source" != "-" && ! -f "$json_source" ]]; then
|
||||
echo "Error: JSON file not found: $json_source" >&2
|
||||
exit 2
|
||||
fi
|
||||
if [[ "$json_source" == "-" ]]; then
|
||||
cat
|
||||
else
|
||||
cat "$json_source"
|
||||
fi | BASE_DIR="$BASE_DIR" python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
base_dir = os.environ.get("BASE_DIR")
|
||||
raw = sys.stdin.read().strip()
|
||||
if not raw:
|
||||
print("Error: empty JSON input", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
data = json.loads(raw)
|
||||
access = data.get("access_token")
|
||||
refresh = data.get("refresh_token")
|
||||
if not access:
|
||||
print("Error: access_token missing", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
def decode_exp(token):
|
||||
payload = token.split(".")[1]
|
||||
padding = "=" * (-len(payload) % 4)
|
||||
decoded = base64.urlsafe_b64decode(payload + padding)
|
||||
payload_json = json.loads(decoded)
|
||||
exp = payload_json.get("exp")
|
||||
iat = payload_json.get("iat")
|
||||
if not exp:
|
||||
raise ValueError("exp missing in token payload")
|
||||
return exp, iat
|
||||
|
||||
|
||||
def save_token(token, suffix):
|
||||
exp, iat = decode_exp(token)
|
||||
fname = datetime.fromtimestamp(exp, tz=timezone(timedelta(hours=4))).strftime("%H-%M-%S")
|
||||
path = os.path.join(base_dir, f"{fname}.{suffix}")
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(token)
|
||||
return path, exp, iat
|
||||
|
||||
access_path, access_exp, access_iat = save_token(access, "token")
|
||||
print(access_path)
|
||||
if refresh:
|
||||
refresh_path, refresh_exp, refresh_iat = save_token(refresh, "refresh")
|
||||
print(refresh_path)
|
||||
PY
|
||||
;;
|
||||
refresh)
|
||||
BASE_DIR="$BASE_DIR" TOKEN_URL="$TOKEN_URL" CLIENT_ID="$CLIENT_ID" THRESHOLD_MIN="$THRESHOLD_MIN" FORCE_REFRESH="$FORCE_REFRESH" python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from urllib import request, parse
|
||||
|
||||
base_dir = os.environ.get("BASE_DIR")
|
||||
url = os.environ.get("TOKEN_URL")
|
||||
client_id = os.environ.get("CLIENT_ID")
|
||||
threshold_min = int(os.environ.get("THRESHOLD_MIN", "10"))
|
||||
force_refresh = os.environ.get("FORCE_REFRESH", "0") == "1"
|
||||
|
||||
|
||||
def decode_exp(token):
|
||||
payload = token.split(".")[1]
|
||||
padding = "=" * (-len(payload) % 4)
|
||||
decoded = base64.urlsafe_b64decode(payload + padding)
|
||||
payload_json = json.loads(decoded)
|
||||
exp = payload_json.get("exp")
|
||||
iat = payload_json.get("iat")
|
||||
if not exp:
|
||||
raise ValueError("exp missing in token payload")
|
||||
return exp, iat
|
||||
|
||||
|
||||
def newest_file(pattern):
|
||||
import glob
|
||||
files = glob.glob(pattern)
|
||||
if not files:
|
||||
return None
|
||||
files.sort(key=lambda p: os.path.getmtime(p), reverse=True)
|
||||
return files[0]
|
||||
|
||||
access_path = newest_file(os.path.join(base_dir, "*.token"))
|
||||
refresh_path = newest_file(os.path.join(base_dir, "*.refresh"))
|
||||
if not refresh_path:
|
||||
print("Error: no refresh token file found", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
with open(refresh_path, "r", encoding="utf-8") as f:
|
||||
refresh_token = f.read().strip()
|
||||
|
||||
refresh_exp, refresh_iat = decode_exp(refresh_token)
|
||||
now = int(datetime.now(tz=timezone.utc).timestamp())
|
||||
if refresh_exp <= now:
|
||||
print("Error: refresh token expired", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
if access_path:
|
||||
with open(access_path, "r", encoding="utf-8") as f:
|
||||
access_token = f.read().strip()
|
||||
access_exp, access_iat = decode_exp(access_token)
|
||||
if not force_refresh and access_exp - now > threshold_min * 60:
|
||||
fname = datetime.fromtimestamp(access_exp, tz=timezone(timedelta(hours=4))).strftime("%H-%M-%S")
|
||||
print(access_path)
|
||||
print(f"Access token valid until {fname} GMT+4; no refresh needed")
|
||||
sys.exit(0)
|
||||
|
||||
post_data = parse.urlencode({
|
||||
"grant_type": "refresh_token",
|
||||
"client_id": client_id,
|
||||
"refresh_token": refresh_token,
|
||||
}).encode("utf-8")
|
||||
req = request.Request(url, data=post_data, method="POST")
|
||||
req.add_header("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
try:
|
||||
with request.urlopen(req, timeout=30) as resp:
|
||||
raw = resp.read().decode("utf-8")
|
||||
except Exception as exc:
|
||||
print("HTTP error:", exc, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except Exception:
|
||||
print("Failed to parse response as JSON", file=sys.stderr)
|
||||
print(raw, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if "access_token" not in data:
|
||||
print("Refresh failed:", data.get("error"), "-", data.get("error_description"), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
access_token = data.get("access_token")
|
||||
new_refresh_token = data.get("refresh_token")
|
||||
|
||||
access_exp, access_iat = decode_exp(access_token)
|
||||
access_fname = datetime.fromtimestamp(access_exp, tz=timezone(timedelta(hours=4))).strftime("%H-%M-%S")
|
||||
access_out = os.path.join(base_dir, f"{access_fname}.token")
|
||||
with open(access_out, "w", encoding="utf-8") as f:
|
||||
f.write(access_token)
|
||||
print(access_out)
|
||||
|
||||
if new_refresh_token:
|
||||
refresh_exp, refresh_iat = decode_exp(new_refresh_token)
|
||||
refresh_fname = datetime.fromtimestamp(refresh_exp, tz=timezone(timedelta(hours=4))).strftime("%H-%M-%S")
|
||||
refresh_out = os.path.join(base_dir, f"{refresh_fname}.refresh")
|
||||
with open(refresh_out, "w", encoding="utf-8") as f:
|
||||
f.write(new_refresh_token)
|
||||
print(refresh_out)
|
||||
PY
|
||||
;;
|
||||
current)
|
||||
BASE_DIR="$BASE_DIR" python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
base_dir = os.environ.get("BASE_DIR")
|
||||
|
||||
def decode_exp(token):
|
||||
payload = token.split(".")[1]
|
||||
padding = "=" * (-len(payload) % 4)
|
||||
decoded = base64.urlsafe_b64decode(payload + padding)
|
||||
payload_json = json.loads(decoded)
|
||||
exp = payload_json.get("exp")
|
||||
iat = payload_json.get("iat")
|
||||
if not exp:
|
||||
raise ValueError("exp missing in token payload")
|
||||
return exp, iat
|
||||
|
||||
paths = [p for p in os.listdir(base_dir) if p.endswith(".token")]
|
||||
if not paths:
|
||||
print("No access token files found")
|
||||
raise SystemExit(2)
|
||||
paths.sort(key=lambda p: os.path.getmtime(os.path.join(base_dir, p)), reverse=True)
|
||||
path = os.path.join(base_dir, paths[0])
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
token = f.read().strip()
|
||||
exp, iat = decode_exp(token)
|
||||
print(path)
|
||||
print("iat", datetime.fromtimestamp(iat, tz=timezone(timedelta(hours=4))).strftime("%Y-%m-%d %H:%M:%S GMT+4"))
|
||||
print("exp", datetime.fromtimestamp(exp, tz=timezone(timedelta(hours=4))).strftime("%Y-%m-%d %H:%M:%S GMT+4"))
|
||||
PY
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
Executable
+270
@@ -0,0 +1,270 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
PROVIDER_DIR="${ROOT_DIR}/provider"
|
||||
PROFILE_DIR="${PROFILE_DIR:-}"
|
||||
|
||||
if [[ "${1:-}" == "--profile" ]]; then
|
||||
PROFILE_DIR="${2:-}"
|
||||
shift 2
|
||||
fi
|
||||
|
||||
if [[ -n "$PROFILE_DIR" ]]; then
|
||||
if [[ ! -d "$PROFILE_DIR" ]]; then
|
||||
echo "Error: profile directory not found: $PROFILE_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env"
|
||||
if [[ -f "$PROFILE_ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$PROFILE_ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$PROFILE_DIR" ]]; then
|
||||
echo "Error: --profile <path> is required. Strict mode stores generated files per stand only." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
resolve_root_path() {
|
||||
local path_value="$1"
|
||||
if [[ -z "$path_value" ]]; then
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
if [[ "$path_value" = /* ]]; then
|
||||
echo "$path_value"
|
||||
return
|
||||
fi
|
||||
echo "${ROOT_DIR}/${path_value}"
|
||||
}
|
||||
|
||||
SERVICES_FILE_DEFAULT="${PROFILE_DIR}/services_list.txt"
|
||||
if [[ -f "${PROFILE_DIR}/services_list.txt" ]]; then
|
||||
SERVICES_FILE_DEFAULT="${PROFILE_DIR}/services_list.txt"
|
||||
fi
|
||||
SERVICES_FILE="${SERVICES_FILE:-$SERVICES_FILE_DEFAULT}"
|
||||
SERVICES_FILE="$(resolve_root_path "$SERVICES_FILE")"
|
||||
|
||||
# Retry behavior (per service): attempts and delay between attempts.
|
||||
REQUEST_DELAY="${REQUEST_DELAY:-0.5}"
|
||||
ATTEMPTS="${ATTEMPTS:-3}"
|
||||
|
||||
# Auth: either an explicit token or a token file (latest *.token if not set).
|
||||
TOKEN_FILE="${TOKEN_FILE:-}"
|
||||
NUBES_API_TOKEN="${NUBES_API_TOKEN:-}"
|
||||
if [[ -n "$TOKEN_FILE" ]]; then
|
||||
TOKEN_FILE="$(resolve_root_path "$TOKEN_FILE")"
|
||||
fi
|
||||
|
||||
# Назначение:
|
||||
# - Берет список сервисов из services_list.txt.
|
||||
# - Для каждого сервиса запрашивает единый spec через API и пишет YAML в
|
||||
# ${ROOT_DIR}/provider/resources_yaml.
|
||||
# - Формат имени файла: ID_имя.yaml (например, 13_s3bucket.yaml).
|
||||
#
|
||||
# Источники данных:
|
||||
# - Список сервисов: TOOLS/config/{stand}/services_list.txt
|
||||
# - API endpoint: ${API_ENDPOINT}
|
||||
# - Токен: переменная NUBES_API_TOKEN или файл .token.
|
||||
#
|
||||
# Формат services_list.txt:
|
||||
# - В начале строки: service_id.
|
||||
# - Остальное в строке — произвольная заметка для девопса (не используется).
|
||||
#
|
||||
# Задержки и попытки:
|
||||
# - REQUEST_DELAY (секунды) между запросами и ретраями (по умолчанию 0.2).
|
||||
# - ATTEMPTS — число попыток на сервис (по умолчанию 3).
|
||||
#
|
||||
# Логи ошибок:
|
||||
# - Список неуспешных сервисов пишется в /tmp/yaml_gen_failures.txt.
|
||||
#
|
||||
# Токен берется из переменной NUBES_API_TOKEN.
|
||||
# Если переменная пуста, берется файл .token из ${ROOT_DIR}:
|
||||
# - "последний" = файл с самым новым временем изменения (ls -t | head -n 1).
|
||||
if [[ -z "$NUBES_API_TOKEN" ]]; then
|
||||
if [[ -z "$TOKEN_FILE" ]]; then
|
||||
TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true)
|
||||
fi
|
||||
if [[ -n "$TOKEN_FILE" && -f "$TOKEN_FILE" ]]; then
|
||||
NUBES_API_TOKEN=$(cat "$TOKEN_FILE")
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$NUBES_API_TOKEN" ]]; then
|
||||
echo "Error: NUBES_API_TOKEN or TOKEN_FILE is required" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ! -f "$SERVICES_FILE" ]]; then
|
||||
echo "Error: services file not found: $SERVICES_FILE" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
API_ENDPOINT="${NUBES_API_ENDPOINT:-https://deck-api.ngcloud.ru/api/v1/index.cfm}"
|
||||
# Auto-detect API style: if endpoint contains "index.cfm" → legacy proxy (?endpoint=),
|
||||
# otherwise → new REST gateway (direct paths). No forced /index.cfm normalization.
|
||||
|
||||
GENERATED_DIR="${ROOT_DIR}/generated/$(basename "$PROFILE_DIR")"
|
||||
YAML_OUTPUT_DIR_DEFAULT="${GENERATED_DIR}/resources_yaml"
|
||||
FAILURES_FILE_DEFAULT="${GENERATED_DIR}/tmp/yaml_gen_failures.txt"
|
||||
|
||||
YAML_OUTPUT_DIR="${YAML_OUTPUT_DIR:-$YAML_OUTPUT_DIR_DEFAULT}"
|
||||
FAILURES_FILE="${FAILURES_FILE:-$FAILURES_FILE_DEFAULT}"
|
||||
YAML_OUTPUT_DIR="$(resolve_root_path "$YAML_OUTPUT_DIR")"
|
||||
FAILURES_FILE="$(resolve_root_path "$FAILURES_FILE")"
|
||||
|
||||
# Убедимся, что папка есть. Полной очистки нет: обрабатываем только сервисы из списка.
|
||||
mkdir -p "$YAML_OUTPUT_DIR"
|
||||
mkdir -p "$(dirname "$FAILURES_FILE")"
|
||||
|
||||
if [[ ! -f "${YAML_OUTPUT_DIR}/embed.go" ]]; then
|
||||
cat > "${YAML_OUTPUT_DIR}/embed.go" <<'EOF'
|
||||
package resources_yaml
|
||||
|
||||
import "embed"
|
||||
|
||||
// Files embeds unified service YAML specs for provider runtime lookup.
|
||||
//
|
||||
//go:embed *.yaml
|
||||
var Files embed.FS
|
||||
EOF
|
||||
fi
|
||||
|
||||
rm -f "$FAILURES_FILE"
|
||||
|
||||
# Build service_spec_gen binary if missing.
|
||||
if [[ ! -x "${ROOT_DIR}/TOOLS/bin/yaml-generator" ]]; then
|
||||
echo "Building service_spec_gen..."
|
||||
# Binary pre-built in TOOLS/bin/yaml-generator
|
||||
fi
|
||||
|
||||
# Проверка что список сервисов не пустой
|
||||
svc_count=$(grep -cv '^\s*$\|^\s*#' "$SERVICES_FILE" || true)
|
||||
if [[ "$svc_count" -eq 0 ]]; then
|
||||
echo "Error: services list is empty or all commented out: $SERVICES_FILE" >&2
|
||||
exit 2
|
||||
fi
|
||||
echo "Generating YAML for ${svc_count} services from ${SERVICES_FILE}"
|
||||
|
||||
# Читаем список сервисов, пропуская пустые строки и комментарии.
|
||||
while IFS= read -r line; do
|
||||
line="${line//$'\r'/}"
|
||||
if [[ -z "$line" ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ "$line" == \#* ]]; then
|
||||
continue
|
||||
fi
|
||||
sid=$(echo "$line" | awk '{print $1}')
|
||||
svc_name=""
|
||||
|
||||
if [[ -z "$sid" ]]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ ! "$sid" =~ ^[0-9]+$ ]]; then
|
||||
echo "Error: invalid service id '${sid}' in ${SERVICES_FILE}" >&2
|
||||
echo "${sid} invalid_id" >> "$FAILURES_FILE"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Если имя не указано в списке — подтягиваем по API.
|
||||
if [[ -z "$svc_name" ]]; then
|
||||
svc_name=$(python3 - <<PY
|
||||
import json, sys, time, urllib.parse, urllib.request
|
||||
|
||||
sid = "${sid}"
|
||||
endpoint = "${API_ENDPOINT}"
|
||||
token = "${NUBES_API_TOKEN}"
|
||||
max_retries = 3
|
||||
|
||||
# Auto-detect API style: "index.cfm" → legacy proxy, otherwise → REST gateway.
|
||||
if "index.cfm" in endpoint:
|
||||
params = urllib.parse.urlencode({"endpoint": f"/services/{sid}"})
|
||||
url = f"{endpoint}?{params}"
|
||||
else:
|
||||
url = f"{endpoint}/services/{sid}"
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
if token:
|
||||
req.add_header("Authorization", f"Bearer {token}")
|
||||
req.add_header("User-Agent", "Mozilla/5.0 (compatible; Terraform-Provider-Nubes/BashGenerator)")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
svc = data.get("svc", {})
|
||||
name = svc.get("svcShort") or svc.get("name") or svc.get("title") or f"service_{sid}"
|
||||
print(name)
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
if attempt < max_retries:
|
||||
time.sleep(2 * attempt)
|
||||
else:
|
||||
print(f"ERROR: failed to fetch service {sid} after {max_retries} attempts: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
PY
|
||||
)
|
||||
if [[ "$svc_name" == ERROR:* ]]; then
|
||||
echo "$svc_name" >&2
|
||||
echo "${sid} api_error" >> "$FAILURES_FILE"
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Один YAML на сервис формируется Go-генератором.
|
||||
echo "Generating unified spec for ${sid} (${svc_name})"
|
||||
success=0
|
||||
# Generator normalizes names, so verify output by ID prefix only.
|
||||
output_glob="${YAML_OUTPUT_DIR}/${sid}_*.yaml"
|
||||
|
||||
# Для выбранного сервиса удаляем старый YAML и генерируем заново с ретраями.
|
||||
rm -f "$output_glob"
|
||||
|
||||
# Генерация YAML для выбранного сервиса с ретраями.
|
||||
for attempt in $(seq 1 "$ATTEMPTS"); do
|
||||
(
|
||||
cd "$PROVIDER_DIR"
|
||||
NUBES_API_TOKEN="$NUBES_API_TOKEN" \
|
||||
NUBES_SERVICE_ID="$sid" \
|
||||
NUBES_SERVICE_NAME="$svc_name" \
|
||||
NUBES_API_ENDPOINT="$API_ENDPOINT" \
|
||||
NUBES_OUTPUT_DIR="$YAML_OUTPUT_DIR" \
|
||||
"${ROOT_DIR}/TOOLS/bin/yaml-generator"
|
||||
) && success=1 || success=0
|
||||
|
||||
if [[ "$success" -eq 1 ]]; then
|
||||
# Success means output file exists for the service ID.
|
||||
if ! compgen -G "$output_glob" > /dev/null; then
|
||||
echo "Error: output not created ${output_glob} for ${sid} (${svc_name})" >&2
|
||||
success=0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$success" -eq 1 ]]; then
|
||||
break
|
||||
fi
|
||||
echo "Retry ${attempt}/${ATTEMPTS} failed for ${sid} (${svc_name})" >&2
|
||||
sleep "$REQUEST_DELAY"
|
||||
done
|
||||
|
||||
# Фиксируем неудачные сервисы для анализа.
|
||||
if [[ "$success" -ne 1 ]]; then
|
||||
echo "${sid} ${svc_name}" >> "$FAILURES_FILE"
|
||||
fi
|
||||
|
||||
sleep "$REQUEST_DELAY"
|
||||
|
||||
done < "$SERVICES_FILE"
|
||||
|
||||
echo "Done. YAML files are in ${YAML_OUTPUT_DIR}"
|
||||
if [[ -f "$FAILURES_FILE" ]]; then
|
||||
echo "Failed services (see $FAILURES_FILE):" >&2
|
||||
cat "$FAILURES_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Generate docs using ClickHouse template (v2, no color markup).
|
||||
# Canonical docs generator for unified resources_yaml.
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
PROVIDER_DIR="${ROOT_DIR}/provider"
|
||||
RESOURCES_YAML_DIR="${PROVIDER_DIR}/resources_yaml"
|
||||
DOCS_DIR="${ROOT_DIR}/docs/30_registry/resources"
|
||||
SERVICES_LIST_PATH="${PROFILE_DIR}/services_list.txt"
|
||||
|
||||
cd "$PROVIDER_DIR"
|
||||
|
||||
${ROOT_DIR}/TOOLS/bin/docs-generator \
|
||||
-resources "$RESOURCES_YAML_DIR" \
|
||||
-docs "$DOCS_DIR" \
|
||||
-services "$SERVICES_LIST_PATH" \
|
||||
-exclude "clickhouse"
|
||||
|
||||
echo "Docs generated in ${DOCS_DIR}"
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Canonical resources+docs generator (template format).
|
||||
# Restores instruction-style docs structure (manual/example/params_create/params_modify/outputs/ops/params landing).
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
PROVIDER_DIR="${ROOT_DIR}/provider"
|
||||
PROFILE_DIR="${PROFILE_DIR:-}"
|
||||
|
||||
resolve_root_path() {
|
||||
local path_value="$1"
|
||||
if [[ -z "$path_value" ]]; then
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
if [[ "$path_value" = /* ]]; then
|
||||
echo "$path_value"
|
||||
return
|
||||
fi
|
||||
echo "${ROOT_DIR}/${path_value}"
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "--profile" ]]; then
|
||||
PROFILE_DIR="${2:-}"
|
||||
shift 2
|
||||
fi
|
||||
|
||||
if [[ -n "$PROFILE_DIR" ]]; then
|
||||
if [[ ! -d "$PROFILE_DIR" ]]; then
|
||||
echo "Error: profile directory not found: $PROFILE_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env"
|
||||
if [[ -f "$PROFILE_ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$PROFILE_ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$PROFILE_DIR" ]]; then
|
||||
echo "Error: --profile <path> is required. Strict mode stores generated files per stand only." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
PROFILE_DIR="$(resolve_root_path "$PROFILE_DIR")"
|
||||
|
||||
RESOURCES_YAML_DIR_DEFAULT="${ROOT_DIR}/generated/$(basename "$PROFILE_DIR")/resources_yaml"
|
||||
DOCS_DIR_DEFAULT="${ROOT_DIR}/generated/$(basename "$PROFILE_DIR")/docs"
|
||||
SERVICES_LIST_PATH_DEFAULT="${PROFILE_DIR}/services_list.txt"
|
||||
GO_OUTPUT_DIR_DEFAULT="${ROOT_DIR}/generated/$(basename "$PROFILE_DIR")/go"
|
||||
|
||||
RESOURCES_YAML_DIR="${RESOURCES_YAML_DIR:-$RESOURCES_YAML_DIR_DEFAULT}"
|
||||
DOCS_DIR="${DOCS_DIR:-$DOCS_DIR_DEFAULT}"
|
||||
SERVICES_LIST_PATH="${SERVICES_LIST_PATH:-$SERVICES_LIST_PATH_DEFAULT}"
|
||||
GO_OUTPUT_DIR="${GO_OUTPUT_DIR:-$GO_OUTPUT_DIR_DEFAULT}"
|
||||
|
||||
RESOURCES_YAML_DIR="$(resolve_root_path "$RESOURCES_YAML_DIR")"
|
||||
DOCS_DIR="$(resolve_root_path "$DOCS_DIR")"
|
||||
SERVICES_LIST_PATH="$(resolve_root_path "$SERVICES_LIST_PATH")"
|
||||
GO_OUTPUT_DIR="$(resolve_root_path "$GO_OUTPUT_DIR")"
|
||||
|
||||
if [[ ! -d "$RESOURCES_YAML_DIR" ]]; then
|
||||
echo "Error: resources yaml directory not found: $RESOURCES_YAML_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ! -f "$SERVICES_LIST_PATH" ]]; then
|
||||
echo "Error: services list not found: $SERVICES_LIST_PATH" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
cd "$PROVIDER_DIR"
|
||||
|
||||
echo "Generating resources from unified YAML specs..."
|
||||
TMP_GEN_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_GEN_DIR"' EXIT
|
||||
|
||||
NUBES_RESOURCES_DIR="$RESOURCES_YAML_DIR" \
|
||||
NUBES_RESOURCES_GEN_DIR="$TMP_GEN_DIR" \
|
||||
${ROOT_DIR}/TOOLS/bin/resource-generator
|
||||
|
||||
mkdir -p "$GO_OUTPUT_DIR"
|
||||
rm -rf "${GO_OUTPUT_DIR}"/*
|
||||
cp -R "$TMP_GEN_DIR/." "$GO_OUTPUT_DIR/"
|
||||
|
||||
echo "Generating docs via template generator..."
|
||||
mkdir -p "$DOCS_DIR"
|
||||
|
||||
# Determine API endpoint for doc examples.
|
||||
# Use NUBES_API_ENDPOINT from profile.env, fall back to production default.
|
||||
DOCS_API_ENDPOINT="${NUBES_API_ENDPOINT:-https://deck-api.ngcloud.ru/api/v1/index.cfm}"
|
||||
# If endpoint looks like new REST gateway (no index.cfm), keep as-is.
|
||||
# If it's old-style without index.cfm, append it for backward compat in docs.
|
||||
if [[ "$DOCS_API_ENDPOINT" != *"/index.cfm"* ]] && [[ "$DOCS_API_ENDPOINT" != *"/svc"* ]]; then
|
||||
DOCS_API_ENDPOINT="${DOCS_API_ENDPOINT%/}/index.cfm"
|
||||
fi
|
||||
|
||||
${ROOT_DIR}/TOOLS/bin/docs-generator \
|
||||
-resources "$RESOURCES_YAML_DIR" \
|
||||
-docs "$DOCS_DIR" \
|
||||
-services "$SERVICES_LIST_PATH" \
|
||||
-api-endpoint "$DOCS_API_ENDPOINT"
|
||||
|
||||
echo "Resources and docs generated in template format."
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
PROVIDER_MAIN="${ROOT_DIR}/provider/main.go"
|
||||
BUILD_SCRIPT="${ROOT_DIR}/TOOLS/scripts/build-provider.sh"
|
||||
PROFILE_DIR="${PROFILE_DIR:-}"
|
||||
|
||||
if [[ "${1:-}" == "--profile" ]]; then
|
||||
PROFILE_DIR="${2:-}"
|
||||
shift 2
|
||||
fi
|
||||
|
||||
if [[ -n "$PROFILE_DIR" ]]; then
|
||||
if [[ ! -d "$PROFILE_DIR" ]]; then
|
||||
echo "Error: profile directory not found: $PROFILE_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env"
|
||||
if [[ -f "$PROFILE_ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$PROFILE_ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$PROFILE_DIR" ]]; then
|
||||
echo "Error: --profile <path> is required. Strict mode stores generated files per stand only." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
resolve_root_path() {
|
||||
local path_value="$1"
|
||||
if [[ -z "$path_value" ]]; then
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
if [[ "$path_value" = /* ]]; then
|
||||
echo "$path_value"
|
||||
return
|
||||
fi
|
||||
echo "${ROOT_DIR}/${path_value}"
|
||||
}
|
||||
|
||||
S3CFG_REGISTRY="${S3CFG_REGISTRY:-${ROOT_DIR}/secrets/.s3cfg_registry}"
|
||||
TIMEOUTS_SOURCE="${TIMEOUTS_SOURCE:-${PROFILE_DIR}/operation_timeouts.json}"
|
||||
PROFILE_RESOURCES_YAML_DIR="${ROOT_DIR}/generated/$(basename "$PROFILE_DIR")/resources_yaml"
|
||||
PROFILE_GENERATED_GO_DIR="${ROOT_DIR}/generated/$(basename "$PROFILE_DIR")/go"
|
||||
PROFILE_BUILD_DIR="${ROOT_DIR}/generated/$(basename "$PROFILE_DIR")/provider_build"
|
||||
S3CFG_REGISTRY="$(resolve_root_path "$S3CFG_REGISTRY")"
|
||||
TIMEOUTS_SOURCE="$(resolve_root_path "$TIMEOUTS_SOURCE")"
|
||||
PROFILE_RESOURCES_YAML_DIR="$(resolve_root_path "$PROFILE_RESOURCES_YAML_DIR")"
|
||||
PROFILE_GENERATED_GO_DIR="$(resolve_root_path "$PROFILE_GENERATED_GO_DIR")"
|
||||
PROFILE_BUILD_DIR="$(resolve_root_path "$PROFILE_BUILD_DIR")"
|
||||
|
||||
if [[ -f "${PROFILE_DIR}/operation_timeouts.json" ]]; then
|
||||
TIMEOUTS_SOURCE="${PROFILE_DIR}/operation_timeouts.json"
|
||||
fi
|
||||
|
||||
load_s3cfg_registry() {
|
||||
local cfg="$1"
|
||||
if [[ ! -f "$cfg" ]]; then
|
||||
echo "Error: S3 config not found: $cfg" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
local access_key secret_key host_base use_https endpoint
|
||||
access_key=$(awk -F '=' '/^\s*access_key\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg")
|
||||
secret_key=$(awk -F '=' '/^\s*secret_key\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg")
|
||||
host_base=$(awk -F '=' '/^\s*host_base\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg")
|
||||
use_https=$(awk -F '=' '/^\s*use_https\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print tolower($2); exit}' "$cfg")
|
||||
|
||||
if [[ -z "${S3_ACCESS_KEY:-}" && -n "$access_key" ]]; then
|
||||
export S3_ACCESS_KEY="$access_key"
|
||||
fi
|
||||
if [[ -z "${S3_SECRET_KEY:-}" && -n "$secret_key" ]]; then
|
||||
export S3_SECRET_KEY="$secret_key"
|
||||
fi
|
||||
|
||||
if [[ -z "${S3_ENDPOINT:-}" && -n "$host_base" ]]; then
|
||||
if [[ "$host_base" == http* ]]; then
|
||||
endpoint="$host_base"
|
||||
else
|
||||
if [[ "$use_https" == "false" || "$use_https" == "0" || "$use_https" == "no" ]]; then
|
||||
endpoint="http://${host_base}"
|
||||
else
|
||||
endpoint="https://${host_base}"
|
||||
fi
|
||||
fi
|
||||
export S3_ENDPOINT="$endpoint"
|
||||
fi
|
||||
|
||||
if [[ -z "${S3_ENDPOINT:-}" || -z "${S3_ACCESS_KEY:-}" || -z "${S3_SECRET_KEY:-}" ]]; then
|
||||
echo "Error: S3 credentials are not set (S3_ENDPOINT/S3_ACCESS_KEY/S3_SECRET_KEY)" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
VERSION="${1:-}"
|
||||
if [[ -z "$VERSION" ]]; then
|
||||
VERSION="${PROVIDER_VERSION:-${RELEASE_VERSION:-}}"
|
||||
fi
|
||||
if [[ -z "$VERSION" ]]; then
|
||||
VERSION=$(grep -E 'version string' "$PROVIDER_MAIN" | sed -E 's/.*"([0-9.]+)".*/\1/')
|
||||
fi
|
||||
|
||||
if [[ -z "$VERSION" ]]; then
|
||||
echo "Error: unable to detect version from $PROVIDER_MAIN" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
load_s3cfg_registry "$S3CFG_REGISTRY"
|
||||
|
||||
if [[ ! -f "$TIMEOUTS_SOURCE" ]]; then
|
||||
echo "Error: operation timeouts config not found: $TIMEOUTS_SOURCE" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ! -d "$PROFILE_RESOURCES_YAML_DIR" ]]; then
|
||||
echo "Error: profile resources yaml directory not found: $PROFILE_RESOURCES_YAML_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
if ! compgen -G "${PROFILE_RESOURCES_YAML_DIR}/*.yaml" > /dev/null; then
|
||||
echo "Error: no yaml specs found in profile: $PROFILE_RESOURCES_YAML_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ! -d "$PROFILE_GENERATED_GO_DIR" ]]; then
|
||||
echo "Error: profile generated go directory not found: $PROFILE_GENERATED_GO_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
if ! compgen -G "${PROFILE_GENERATED_GO_DIR}/*.go" > /dev/null; then
|
||||
echo "Error: no generated go files found in profile: $PROFILE_GENERATED_GO_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
validate_registry_completeness() {
|
||||
local resources_gen_dir="$1"
|
||||
local registry_file="${resources_gen_dir}/registry.go"
|
||||
if [[ ! -f "$registry_file" ]]; then
|
||||
echo "Error: registry.go not found in resources_gen: $resources_gen_dir" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
local tmp_expected tmp_registered tmp_missing
|
||||
tmp_expected="$(mktemp)"
|
||||
tmp_registered="$(mktemp)"
|
||||
tmp_missing="$(mktemp)"
|
||||
|
||||
find "$resources_gen_dir" -maxdepth 1 -type f -name '*_resource.go' -print0 \
|
||||
| xargs -0 grep -hE 'func[[:space:]]+New[[:alnum:]_]+Resource[[:space:]]*\(' \
|
||||
| sed -E 's/.*(New[[:alnum:]_]+Resource).*/\1/' \
|
||||
| sort -u > "$tmp_expected"
|
||||
|
||||
grep -hEo 'New[[:alnum:]_]+Resource' "$registry_file" | sort -u > "$tmp_registered"
|
||||
|
||||
comm -23 "$tmp_expected" "$tmp_registered" > "$tmp_missing"
|
||||
|
||||
if [[ -s "$tmp_missing" ]]; then
|
||||
echo "Error: registry.go is missing resource constructors:" >&2
|
||||
sed 's/^/ - /' "$tmp_missing" >&2
|
||||
rm -f "$tmp_expected" "$tmp_registered" "$tmp_missing"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
rm -f "$tmp_expected" "$tmp_registered" "$tmp_missing"
|
||||
}
|
||||
|
||||
TMP_PROVIDER_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_PROVIDER_DIR"' EXIT
|
||||
|
||||
cp -a "${ROOT_DIR}/provider/." "$TMP_PROVIDER_DIR/"
|
||||
mkdir -p "$TMP_PROVIDER_DIR/resources_yaml"
|
||||
find "$TMP_PROVIDER_DIR/resources_yaml" -maxdepth 1 -type f -name '*.yaml' -delete
|
||||
cp -a "${PROFILE_RESOURCES_YAML_DIR}"/*.yaml "$TMP_PROVIDER_DIR/resources_yaml/"
|
||||
cp "$TIMEOUTS_SOURCE" "$TMP_PROVIDER_DIR/internal/provider/operation_timeouts.json"
|
||||
|
||||
mkdir -p "$TMP_PROVIDER_DIR/internal/resources_gen"
|
||||
find "$TMP_PROVIDER_DIR/internal/resources_gen" -maxdepth 1 -type f -name '*.go' -delete
|
||||
cp -R "${PROFILE_GENERATED_GO_DIR}"/*.go "$TMP_PROVIDER_DIR/internal/resources_gen/"
|
||||
|
||||
validate_registry_completeness "$TMP_PROVIDER_DIR/internal/resources_gen"
|
||||
|
||||
mkdir -p "$PROFILE_BUILD_DIR"
|
||||
|
||||
if [[ -n "${GPG_PRIVATE_KEY_PATH:-}" ]]; then
|
||||
export GPG_KEY_FILE="$(resolve_root_path "$GPG_PRIVATE_KEY_PATH")"
|
||||
fi
|
||||
|
||||
echo "Building provider version ${VERSION}..."
|
||||
PROVIDER_DIR="$TMP_PROVIDER_DIR" BUILD_DIR="$PROFILE_BUILD_DIR" "$BUILD_SCRIPT" "$VERSION"
|
||||
Executable
+248
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
PROVIDER_MAIN="${ROOT_DIR}/provider/main.go"
|
||||
PROFILE_DIR="${PROFILE_DIR:-}"
|
||||
|
||||
normalize_api_endpoint() {
|
||||
local endpoint="$1"
|
||||
endpoint="${endpoint%/}"
|
||||
if [[ -z "$endpoint" ]]; then
|
||||
echo "https://deck-api.ngcloud.ru/api/v1/index.cfm"
|
||||
return
|
||||
fi
|
||||
# Keep as-is: both legacy proxy (index.cfm) and new REST gateway are valid.
|
||||
# No forced /index.cfm normalization — the downstream tools auto-detect.
|
||||
echo "$endpoint"
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "--profile" ]]; then
|
||||
PROFILE_DIR="${2:-}"
|
||||
shift 2
|
||||
fi
|
||||
|
||||
if [[ -n "$PROFILE_DIR" ]]; then
|
||||
if [[ ! -d "$PROFILE_DIR" ]]; then
|
||||
echo "Error: profile directory not found: $PROFILE_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env"
|
||||
if [[ -f "$PROFILE_ENV_FILE" ]]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$PROFILE_ENV_FILE"
|
||||
set +a
|
||||
fi
|
||||
fi
|
||||
|
||||
resolve_root_path() {
|
||||
local path_value="$1"
|
||||
if [[ -z "$path_value" ]]; then
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
if [[ "$path_value" = /* ]]; then
|
||||
echo "$path_value"
|
||||
return
|
||||
fi
|
||||
echo "${ROOT_DIR}/${path_value}"
|
||||
}
|
||||
|
||||
VERSION="${1:-}"
|
||||
if [[ -z "$VERSION" ]]; then
|
||||
VERSION="${DOCS_VERSION:-${RELEASE_VERSION:-}}"
|
||||
fi
|
||||
if [[ -z "$VERSION" ]]; then
|
||||
VERSION=$(grep -E 'version string' "$PROVIDER_MAIN" | sed -E 's/.*"([0-9.]+)".*/\1/')
|
||||
fi
|
||||
|
||||
if [[ -z "$VERSION" ]]; then
|
||||
echo "Error: unable to detect version from $PROVIDER_MAIN" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
REGISTRY_HOST="${REGISTRY_HOST:-terra.k8c.ru}"
|
||||
NAMESPACE="${NAMESPACE:-nubes}"
|
||||
PROVIDER_NAME="${PROVIDER_NAME:-nubes}"
|
||||
export REGISTRY_HOST NAMESPACE PROVIDER_NAME VERSION
|
||||
|
||||
S3CFG_REGISTRY="${S3CFG_REGISTRY:-${ROOT_DIR}/secrets/.s3cfg_registry}"
|
||||
S3CFG_REGISTRY="$(resolve_root_path "$S3CFG_REGISTRY")"
|
||||
TMP_DOCS_DIR=""
|
||||
|
||||
if [[ -n "$PROFILE_DIR" ]]; then
|
||||
# ⛔ NEVER merge with docs/ — ONLY generated docs from docs_gen/<stand>/
|
||||
DOCS_GEN_DIR="${DOCS_GEN_DIR:-generated/test}"
|
||||
DOCS_GEN_DIR="$(resolve_root_path "$DOCS_GEN_DIR")"
|
||||
if [[ -d "$DOCS_GEN_DIR" ]]; then
|
||||
export MKDOCS_DOCS_DIR="$DOCS_GEN_DIR"
|
||||
else
|
||||
echo "Error: DOCS_GEN_DIR not found: $DOCS_GEN_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
fi
|
||||
|
||||
DOCS_API_ENDPOINT="${NUBES_API_ENDPOINT:-https://deck-api.ngcloud.ru/api/v1/index.cfm}"
|
||||
DOCS_API_ENDPOINT="$(normalize_api_endpoint "$DOCS_API_ENDPOINT")"
|
||||
|
||||
if [[ -n "${MKDOCS_DOCS_DIR:-}" ]]; then
|
||||
export DOCS_GUIDE_VERSION="$VERSION"
|
||||
export DOCS_GUIDE_API_ENDPOINT="$DOCS_API_ENDPOINT"
|
||||
export DOCS_GUIDE_FILE="${MKDOCS_DOCS_DIR}/guides/getting-started.md"
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
guide_path = Path(os.environ["DOCS_GUIDE_FILE"])
|
||||
if guide_path.exists():
|
||||
text = guide_path.read_text(encoding="utf-8")
|
||||
text = re.sub(
|
||||
r'(version\s*=\s*")([0-9.]+)("\s*#\s*Поставьте нужную вам версию провайдера)',
|
||||
lambda m: f'{m.group(1)}{os.environ["DOCS_GUIDE_VERSION"]}{m.group(3)}',
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
text = re.sub(
|
||||
r'(api_endpoint\s*=\s*")([^"]+)(")',
|
||||
lambda m: f'{m.group(1)}{os.environ["DOCS_GUIDE_API_ENDPOINT"]}{m.group(3)}',
|
||||
text,
|
||||
count=1,
|
||||
)
|
||||
guide_path.write_text(text, encoding="utf-8")
|
||||
PY
|
||||
fi
|
||||
|
||||
load_s3cfg_registry() {
|
||||
local cfg="$1"
|
||||
if [[ ! -f "$cfg" ]]; then
|
||||
echo "Error: S3 config not found: $cfg" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
local access_key secret_key host_base use_https endpoint
|
||||
access_key=$(awk -F '=' '/^\s*access_key\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg")
|
||||
secret_key=$(awk -F '=' '/^\s*secret_key\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg")
|
||||
host_base=$(awk -F '=' '/^\s*host_base\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg")
|
||||
use_https=$(awk -F '=' '/^\s*use_https\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print tolower($2); exit}' "$cfg")
|
||||
|
||||
if [[ -z "${S3_ACCESS_KEY:-}" && -n "$access_key" ]]; then
|
||||
export S3_ACCESS_KEY="$access_key"
|
||||
fi
|
||||
if [[ -z "${S3_SECRET_KEY:-}" && -n "$secret_key" ]]; then
|
||||
export S3_SECRET_KEY="$secret_key"
|
||||
fi
|
||||
|
||||
if [[ -z "${S3_ENDPOINT:-}" && -n "$host_base" ]]; then
|
||||
if [[ "$host_base" == http* ]]; then
|
||||
endpoint="$host_base"
|
||||
else
|
||||
if [[ "$use_https" == "false" || "$use_https" == "0" || "$use_https" == "no" ]]; then
|
||||
endpoint="http://${host_base}"
|
||||
else
|
||||
endpoint="https://${host_base}"
|
||||
fi
|
||||
fi
|
||||
export S3_ENDPOINT="$endpoint"
|
||||
fi
|
||||
|
||||
if [[ -z "${S3_ENDPOINT:-}" || -z "${S3_ACCESS_KEY:-}" || -z "${S3_SECRET_KEY:-}" ]]; then
|
||||
echo "Error: S3 credentials are not set (S3_ENDPOINT/S3_ACCESS_KEY/S3_SECRET_KEY)" >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
TMP_CFG="${ROOT_DIR}/.mkdocs.tmp.yml"
|
||||
export ROOT_DIR
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
registry_host = os.environ["REGISTRY_HOST"]
|
||||
namespace = os.environ["NAMESPACE"]
|
||||
name = os.environ["PROVIDER_NAME"]
|
||||
version = os.environ["VERSION"]
|
||||
root_dir = Path(os.environ["ROOT_DIR"])
|
||||
|
||||
cfg_path = root_dir / "mkdocs.yml"
|
||||
text = cfg_path.read_text(encoding="utf-8")
|
||||
text = re.sub(
|
||||
r"^site_url:\s*.*$",
|
||||
f"site_url: https://{registry_host}/docs/{namespace}/{name}/{version}/",
|
||||
text,
|
||||
flags=re.M,
|
||||
)
|
||||
|
||||
docs_dir = os.environ.get("MKDOCS_DOCS_DIR", "").strip()
|
||||
if docs_dir:
|
||||
if re.search(r"^docs_dir:\s*.*$", text, flags=re.M):
|
||||
text = re.sub(r"^docs_dir:\s*.*$", f"docs_dir: {docs_dir}", text, flags=re.M)
|
||||
else:
|
||||
text += f"\ndocs_dir: {docs_dir}\n"
|
||||
|
||||
(root_dir / ".mkdocs.tmp.yml").write_text(text, encoding="utf-8")
|
||||
PY
|
||||
|
||||
build_docs() {
|
||||
local docs_build_mode
|
||||
docs_build_mode="${DOCS_BUILD_MODE:-auto}"
|
||||
|
||||
if [[ -n "${MKDOCS_DOCS_DIR:-}" && "$docs_build_mode" == "auto" ]]; then
|
||||
docs_build_mode="local"
|
||||
fi
|
||||
|
||||
if [[ "$docs_build_mode" != "local" ]] && command -v docker >/dev/null 2>&1; then
|
||||
local docker_err
|
||||
docker_err="$(mktemp)"
|
||||
if docker run --rm -v "${ROOT_DIR}:/docs" squidfunk/mkdocs-material build -f "/docs/.mkdocs.tmp.yml" -d site 2>"$docker_err"; then
|
||||
rm -f "$docker_err"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if grep -q "error while creating mount source path" "$docker_err"; then
|
||||
echo "Warning: docker mount issue detected, using local mkdocs" >&2
|
||||
else
|
||||
cat "$docker_err" >&2
|
||||
echo "Warning: docker build failed, trying local mkdocs" >&2
|
||||
fi
|
||||
rm -f "$docker_err"
|
||||
fi
|
||||
|
||||
local venv_python
|
||||
venv_python="${ROOT_DIR}/.venv/bin/python"
|
||||
|
||||
if [[ -x "$venv_python" ]]; then
|
||||
if "$venv_python" -m mkdocs --version >/dev/null 2>&1; then
|
||||
(cd "$ROOT_DIR" && "$venv_python" -m mkdocs build -f "$TMP_CFG" -d site)
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Warning: mkdocs module is missing in .venv, installing pinned versions" >&2
|
||||
"$venv_python" -m pip install --disable-pip-version-check -q "mkdocs==1.6.1" "mkdocs-material==9.7.3"
|
||||
(cd "$ROOT_DIR" && "$venv_python" -m mkdocs build -f "$TMP_CFG" -d site)
|
||||
return 0
|
||||
fi
|
||||
|
||||
if command -v mkdocs >/dev/null 2>&1 && mkdocs --version >/dev/null 2>&1; then
|
||||
(cd "$ROOT_DIR" && mkdocs build -f "$TMP_CFG" -d site)
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Error: mkdocs is not installed or not runnable." >&2
|
||||
echo "Install once: python3 -m venv .venv && .venv/bin/python -m pip install mkdocs==1.6.1 mkdocs-material==9.7.3" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
trap 'rm -f "$TMP_CFG"' EXIT
|
||||
build_docs
|
||||
|
||||
:
|
||||
|
||||
:
|
||||
|
||||
load_s3cfg_registry "$S3CFG_REGISTRY"
|
||||
(cd "$ROOT_DIR" && ./scripts/publish-docs.sh site "$REGISTRY_HOST" "$NAMESPACE" "$PROVIDER_NAME" "$VERSION")
|
||||
echo "Docs published for ${VERSION}"
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
SCRIPT="${ROOT_DIR}/TOOLS/scripts/01_generate_yamls.sh"
|
||||
TOKEN_FILE="${TOKEN_FILE:-}"
|
||||
REQUEST_DELAY="${REQUEST_DELAY:-0.2}"
|
||||
ATTEMPTS="${ATTEMPTS:-3}"
|
||||
RUNS="${RUNS:-10}"
|
||||
LOG="${LOG:-/tmp/yaml_gen_runs.log}"
|
||||
|
||||
export TOKEN_FILE
|
||||
|
||||
if [[ -z "$TOKEN_FILE" ]]; then
|
||||
TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true)
|
||||
fi
|
||||
|
||||
if [[ -z "$TOKEN_FILE" || ! -f "$TOKEN_FILE" ]]; then
|
||||
echo "Error: token file not found in ${ROOT_DIR}" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
: > "$LOG"
|
||||
|
||||
echo "Runs: $RUNS" | tee -a "$LOG"
|
||||
echo "Request delay: $REQUEST_DELAY" | tee -a "$LOG"
|
||||
echo "Attempts per service: $ATTEMPTS" | tee -a "$LOG"
|
||||
echo "Token file: $TOKEN_FILE" | tee -a "$LOG"
|
||||
|
||||
echo "" | tee -a "$LOG"
|
||||
|
||||
for i in $(seq 1 "$RUNS"); do
|
||||
start=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
echo "RUN $i start $start" | tee -a "$LOG"
|
||||
REQUEST_DELAY="$REQUEST_DELAY" ATTEMPTS="$ATTEMPTS" "$SCRIPT" >> "$LOG" 2>&1
|
||||
rc=$?
|
||||
end=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
echo "RUN $i end $end rc=$rc" | tee -a "$LOG"
|
||||
echo "" | tee -a "$LOG"
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
echo "Log: $LOG" | tee -a "$LOG"
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
TOKEN_FILE="${TOKEN_FILE:-}"
|
||||
|
||||
if [[ -z "$TOKEN_FILE" ]]; then
|
||||
TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true)
|
||||
fi
|
||||
|
||||
if [[ -z "$TOKEN_FILE" || ! -f "$TOKEN_FILE" ]]; then
|
||||
echo "Error: token file not found in ${ROOT_DIR}" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
export TOKEN_FILE
|
||||
exec "${ROOT_DIR}/TOOLS/scripts/10_yaml_stability_run.sh"
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
TOKEN_FILE="${TOKEN_FILE:-}"
|
||||
|
||||
if [[ -z "$TOKEN_FILE" ]]; then
|
||||
TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true)
|
||||
fi
|
||||
|
||||
if [[ -z "$TOKEN_FILE" || ! -f "$TOKEN_FILE" ]]; then
|
||||
echo "Error: token file not found in ${ROOT_DIR}" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
export TOKEN_FILE
|
||||
exec "${ROOT_DIR}/TOOLS/scripts/01_generate_yamls.sh"
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
PROVIDER_DIR="${ROOT_DIR}/provider"
|
||||
|
||||
if [[ ! -d "${PROVIDER_DIR}/resources_yaml" ]]; then
|
||||
echo "Error: resources_yaml not found: ${PROVIDER_DIR}/resources_yaml" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Start from a clean slate to avoid stale YAMLs.
|
||||
rm -f "${PROVIDER_DIR}/resources_yaml"/*.yaml
|
||||
|
||||
exec "${ROOT_DIR}/TOOLS/scripts/12_generate_yamls_latest.sh"
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build and upload the Terraform provider to the registry S3 bucket.
|
||||
#
|
||||
# GPG signing key requirements:
|
||||
# - A private key is required for signing SHA256SUMS.
|
||||
# - The private key must be available at:
|
||||
# ${ROOT_DIR}/secrets/private_key.asc
|
||||
# - The private key is intentionally ignored by git.
|
||||
#
|
||||
# If the key is missing, generate it with a no-passphrase batch run, then
|
||||
# export it to the expected location. Example (email only, no passphrase):
|
||||
# GPG_DIR=${ROOT_DIR}/secrets
|
||||
# GNUPGHOME=$(mktemp -d)
|
||||
# cat > /tmp/gpg_batch <<'EOF'
|
||||
# %no-protection
|
||||
# Key-Type: RSA
|
||||
# Key-Length: 3072
|
||||
# Subkey-Type: RSA
|
||||
# Subkey-Length: 3072
|
||||
# Name-Real: tazet@narod.ru
|
||||
# Name-Email: tazet@narod.ru
|
||||
# Expire-Date: 0
|
||||
# EOF
|
||||
# gpg --batch --homedir "$GNUPGHOME" --gen-key /tmp/gpg_batch
|
||||
# gpg --batch --homedir "$GNUPGHOME" --armor --export-secret-keys > "$GPG_DIR/private_key.asc"
|
||||
# gpg --batch --homedir "$GNUPGHOME" --armor --export > "$GPG_DIR/public_key.asc"
|
||||
# rm -rf "$GNUPGHOME" /tmp/gpg_batch
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/../.." && pwd)}"
|
||||
PROVIDER_DIR="${PROVIDER_DIR:-${ROOT_DIR}/provider}"
|
||||
BUILD_DIR="${BUILD_DIR:-${ROOT_DIR}/artifacts/provider_build}"
|
||||
GPG_KEY_FILE="${GPG_KEY_FILE:-${ROOT_DIR}/secrets/private_key.asc}"
|
||||
|
||||
VERSION="${1:-2.0.2}"
|
||||
REGISTRY_HOSTNAME="${REGISTRY_HOSTNAME:-terra.k8c.ru}"
|
||||
NAMESPACE="${NAMESPACE:-nubes}"
|
||||
PROVIDER_NAME="${PROVIDER_NAME:-nubes}"
|
||||
S3_BUCKET="${S3_BUCKET:-terraform-registry}"
|
||||
|
||||
S3_ENDPOINT="${S3_ENDPOINT:-}"
|
||||
S3_ACCESS_KEY="${S3_ACCESS_KEY:-}"
|
||||
S3_SECRET_KEY="${S3_SECRET_KEY:-}"
|
||||
|
||||
if [[ -z "$S3_ENDPOINT" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" ]]; then
|
||||
echo "Error: S3_ENDPOINT/S3_ACCESS_KEY/S3_SECRET_KEY must be set" >&2
|
||||
echo "Hint: see ${ROOT_DIR}/secrets/.s3cfg_registry" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ! -f "$GPG_KEY_FILE" ]]; then
|
||||
echo "Error: GPG key file not found at $GPG_KEY_FILE" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ! -d "$PROVIDER_DIR" ]]; then
|
||||
echo "Error: provider source dir not found: $PROVIDER_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
s3_endpoint_url="$S3_ENDPOINT"
|
||||
if [[ "$s3_endpoint_url" != http* ]]; then
|
||||
s3_endpoint_url="https://${s3_endpoint_url}"
|
||||
fi
|
||||
|
||||
rm -rf "$BUILD_DIR"
|
||||
mkdir -p "$BUILD_DIR"
|
||||
|
||||
build_and_zip() {
|
||||
local os="$1"
|
||||
local arch="$2"
|
||||
local binary="terraform-provider-nubes_v${VERSION}"
|
||||
|
||||
if [[ "$os" == "windows" ]]; then
|
||||
binary+=".exe"
|
||||
fi
|
||||
|
||||
echo "Building for ${os}/${arch}..."
|
||||
(cd "$PROVIDER_DIR" && CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -ldflags "-X main.version=${VERSION}" -o "${BUILD_DIR}/${binary}" .)
|
||||
|
||||
(cd "$BUILD_DIR" && python3 -m zipfile -c "terraform-provider-nubes_${VERSION}_${os}_${arch}.zip" "$binary")
|
||||
rm -f "${BUILD_DIR:?}/${binary}"
|
||||
}
|
||||
|
||||
build_and_zip linux amd64
|
||||
build_and_zip windows amd64
|
||||
build_and_zip darwin amd64
|
||||
|
||||
( cd "$BUILD_DIR" && sha256sum *.zip > "terraform-provider-nubes_${VERSION}_SHA256SUMS" )
|
||||
|
||||
GNUPGHOME="$(mktemp -d)"
|
||||
export GNUPGHOME
|
||||
trap 'rm -rf "$GNUPGHOME"' EXIT
|
||||
|
||||
gpg --batch --import "$GPG_KEY_FILE" >/dev/null
|
||||
key_id=$(gpg --list-keys --with-colons | awk -F: '/^pub/{print $5; exit}')
|
||||
if [[ -z "$key_id" ]]; then
|
||||
echo "Error: failed to read key id from $GPG_KEY_FILE" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
gpg --batch --yes --detach-sign --default-key "$key_id" \
|
||||
--output "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS.sig" \
|
||||
"${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS"
|
||||
|
||||
mc alias set registry "$s3_endpoint_url" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" --api S3v4
|
||||
S3_PATH="registry/${S3_BUCKET}/${REGISTRY_HOSTNAME}/${NAMESPACE}/${PROVIDER_NAME}/${VERSION}/"
|
||||
|
||||
echo "Uploading to: ${S3_PATH}"
|
||||
mc cp "${BUILD_DIR}"/*.zip "$S3_PATH"
|
||||
mc cp "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS" "$S3_PATH"
|
||||
mc cp "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS.sig" "$S3_PATH"
|
||||
|
||||
echo "Done. Version ${VERSION} uploaded."
|
||||
Reference in New Issue
Block a user