From 540c1f729396b43808b1403c036ed1ed2c14cf6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Tue, 30 Jun 2026 15:45:16 +0400 Subject: [PATCH] add: devops scripts and configs --- devops/00_token_manager.sh | 230 ++++++++++++++++ devops/01_generate_yamls.sh | 241 ++++++++++++++++ devops/02_generate_resources_and_docs.sh | 24 ++ ...02_generate_resources_and_docs_template.sh | 22 ++ ...generate_resources_and_docs_template_v2.sh | 22 ++ devops/02_generate_resources_and_docs_v2.sh | 98 +++++++ devops/03_build_and_upload_provider.sh | 193 +++++++++++++ devops/04_build_and_publish_docs.sh | 250 +++++++++++++++++ devops/04b_enrich_outputs_from_inventory.py | 151 ++++++++++ devops/10_yaml_stability_run.sh | 44 +++ devops/11_yaml_stability_run_latest.sh | 18 ++ devops/12_generate_yamls_latest.sh | 18 ++ devops/13_generate_yamls_clean.sh | 16 ++ devops/ARCHITECTURE.md | 131 +++++++++ devops/README.md | 258 ++++++++++++++++++ devops/build-provider.sh | 115 ++++++++ devops/ci/pipeline.yaml | 64 +++++ devops/config/operation_timeouts.json | 17 ++ devops/config/services_list.txt | 46 ++++ devops/profiles/dev/operation_timeouts.json | 17 ++ devops/profiles/dev/profile.env | 20 ++ devops/profiles/dev/services_list.txt | 49 ++++ devops/profiles/prod/operation_timeouts.json | 17 ++ devops/profiles/prod/profile.env | 20 ++ devops/profiles/prod/services_list.txt | 46 ++++ devops/profiles/test/operation_timeouts.json | 17 ++ devops/profiles/test/profile.env | 20 ++ devops/profiles/test/services_list.txt | 46 ++++ 28 files changed, 2210 insertions(+) create mode 100755 devops/00_token_manager.sh create mode 100755 devops/01_generate_yamls.sh create mode 100755 devops/02_generate_resources_and_docs.sh create mode 100755 devops/02_generate_resources_and_docs_template.sh create mode 100755 devops/02_generate_resources_and_docs_template_v2.sh create mode 100755 devops/02_generate_resources_and_docs_v2.sh create mode 100755 devops/03_build_and_upload_provider.sh create mode 100755 devops/04_build_and_publish_docs.sh create mode 100644 devops/04b_enrich_outputs_from_inventory.py create mode 100755 devops/10_yaml_stability_run.sh create mode 100755 devops/11_yaml_stability_run_latest.sh create mode 100755 devops/12_generate_yamls_latest.sh create mode 100755 devops/13_generate_yamls_clean.sh create mode 100644 devops/ARCHITECTURE.md create mode 100644 devops/README.md create mode 100755 devops/build-provider.sh create mode 100644 devops/ci/pipeline.yaml create mode 100644 devops/config/operation_timeouts.json create mode 100644 devops/config/services_list.txt create mode 100644 devops/profiles/dev/operation_timeouts.json create mode 100644 devops/profiles/dev/profile.env create mode 100644 devops/profiles/dev/services_list.txt create mode 100644 devops/profiles/prod/operation_timeouts.json create mode 100644 devops/profiles/prod/profile.env create mode 100644 devops/profiles/prod/services_list.txt create mode 100644 devops/profiles/test/operation_timeouts.json create mode 100644 devops/profiles/test/profile.env create mode 100644 devops/profiles/test/services_list.txt diff --git a/devops/00_token_manager.sh b/devops/00_token_manager.sh new file mode 100755 index 0000000..0d88a5a --- /dev/null +++ b/devops/00_token_manager.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEFAULT_BASE_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +BASE_DIR="${BASE_DIR:-${DEFAULT_BASE_DIR}}" +CLIENT_ID="${CLIENT_ID:-account-console}" +TOKEN_URL="${TOKEN_URL:-https://keycloak.nubes.ru/realms/cloud/protocol/openid-connect/token}" +THRESHOLD_MIN="${THRESHOLD_MIN:-10}" +FORCE_REFRESH="${FORCE_REFRESH:-0}" + +usage() { + echo "Usage:" >&2 + echo " $0 save [json_file|-]" >&2 + echo " $0 refresh" >&2 + echo " $0 current" >&2 + echo "" >&2 + echo "Env vars:" >&2 + echo " BASE_DIR=${DEFAULT_BASE_DIR}" >&2 + echo " THRESHOLD_MIN=10" >&2 + echo " FORCE_REFRESH=1" >&2 + echo " TOKEN_URL=https://keycloak.nubes.ru/realms/cloud/protocol/openid-connect/token" >&2 + echo " CLIENT_ID=account-console" >&2 +} + +cmd="${1:-}" + +case "$cmd" in + save) + json_source="${2:--}" + if [[ "$json_source" != "-" && ! -f "$json_source" ]]; then + echo "Error: JSON file not found: $json_source" >&2 + exit 2 + fi + if [[ "$json_source" == "-" ]]; then + cat + else + cat "$json_source" + fi | BASE_DIR="$BASE_DIR" python3 - <<'PY' +import base64 +import json +import os +import sys +from datetime import datetime, timezone, timedelta + +base_dir = os.environ.get("BASE_DIR") +raw = sys.stdin.read().strip() +if not raw: + print("Error: empty JSON input", file=sys.stderr) + sys.exit(2) + +data = json.loads(raw) +access = data.get("access_token") +refresh = data.get("refresh_token") +if not access: + print("Error: access_token missing", file=sys.stderr) + sys.exit(2) + +def decode_exp(token): + payload = token.split(".")[1] + padding = "=" * (-len(payload) % 4) + decoded = base64.urlsafe_b64decode(payload + padding) + payload_json = json.loads(decoded) + exp = payload_json.get("exp") + iat = payload_json.get("iat") + if not exp: + raise ValueError("exp missing in token payload") + return exp, iat + + +def save_token(token, suffix): + exp, iat = decode_exp(token) + fname = datetime.fromtimestamp(exp, tz=timezone(timedelta(hours=4))).strftime("%H-%M-%S") + path = os.path.join(base_dir, f"{fname}.{suffix}") + with open(path, "w", encoding="utf-8") as f: + f.write(token) + return path, exp, iat + +access_path, access_exp, access_iat = save_token(access, "token") +print(access_path) +if refresh: + refresh_path, refresh_exp, refresh_iat = save_token(refresh, "refresh") + print(refresh_path) +PY + ;; + refresh) + BASE_DIR="$BASE_DIR" TOKEN_URL="$TOKEN_URL" CLIENT_ID="$CLIENT_ID" THRESHOLD_MIN="$THRESHOLD_MIN" FORCE_REFRESH="$FORCE_REFRESH" python3 - <<'PY' +import base64 +import json +import os +import sys +from datetime import datetime, timezone, timedelta +from urllib import request, parse + +base_dir = os.environ.get("BASE_DIR") +url = os.environ.get("TOKEN_URL") +client_id = os.environ.get("CLIENT_ID") +threshold_min = int(os.environ.get("THRESHOLD_MIN", "10")) +force_refresh = os.environ.get("FORCE_REFRESH", "0") == "1" + + +def decode_exp(token): + payload = token.split(".")[1] + padding = "=" * (-len(payload) % 4) + decoded = base64.urlsafe_b64decode(payload + padding) + payload_json = json.loads(decoded) + exp = payload_json.get("exp") + iat = payload_json.get("iat") + if not exp: + raise ValueError("exp missing in token payload") + return exp, iat + + +def newest_file(pattern): + import glob + files = glob.glob(pattern) + if not files: + return None + files.sort(key=lambda p: os.path.getmtime(p), reverse=True) + return files[0] + +access_path = newest_file(os.path.join(base_dir, "*.token")) +refresh_path = newest_file(os.path.join(base_dir, "*.refresh")) +if not refresh_path: + print("Error: no refresh token file found", file=sys.stderr) + sys.exit(2) + +with open(refresh_path, "r", encoding="utf-8") as f: + refresh_token = f.read().strip() + +refresh_exp, refresh_iat = decode_exp(refresh_token) +now = int(datetime.now(tz=timezone.utc).timestamp()) +if refresh_exp <= now: + print("Error: refresh token expired", file=sys.stderr) + sys.exit(2) + +if access_path: + with open(access_path, "r", encoding="utf-8") as f: + access_token = f.read().strip() + access_exp, access_iat = decode_exp(access_token) + if not force_refresh and access_exp - now > threshold_min * 60: + fname = datetime.fromtimestamp(access_exp, tz=timezone(timedelta(hours=4))).strftime("%H-%M-%S") + print(access_path) + print(f"Access token valid until {fname} GMT+4; no refresh needed") + sys.exit(0) + +post_data = parse.urlencode({ + "grant_type": "refresh_token", + "client_id": client_id, + "refresh_token": refresh_token, +}).encode("utf-8") +req = request.Request(url, data=post_data, method="POST") +req.add_header("Content-Type", "application/x-www-form-urlencoded") + +try: + with request.urlopen(req, timeout=30) as resp: + raw = resp.read().decode("utf-8") +except Exception as exc: + print("HTTP error:", exc, file=sys.stderr) + sys.exit(1) + +try: + data = json.loads(raw) +except Exception: + print("Failed to parse response as JSON", file=sys.stderr) + print(raw, file=sys.stderr) + sys.exit(1) + +if "access_token" not in data: + print("Refresh failed:", data.get("error"), "-", data.get("error_description"), file=sys.stderr) + sys.exit(1) + +access_token = data.get("access_token") +new_refresh_token = data.get("refresh_token") + +access_exp, access_iat = decode_exp(access_token) +access_fname = datetime.fromtimestamp(access_exp, tz=timezone(timedelta(hours=4))).strftime("%H-%M-%S") +access_out = os.path.join(base_dir, f"{access_fname}.token") +with open(access_out, "w", encoding="utf-8") as f: + f.write(access_token) +print(access_out) + +if new_refresh_token: + refresh_exp, refresh_iat = decode_exp(new_refresh_token) + refresh_fname = datetime.fromtimestamp(refresh_exp, tz=timezone(timedelta(hours=4))).strftime("%H-%M-%S") + refresh_out = os.path.join(base_dir, f"{refresh_fname}.refresh") + with open(refresh_out, "w", encoding="utf-8") as f: + f.write(new_refresh_token) + print(refresh_out) +PY + ;; + current) + BASE_DIR="$BASE_DIR" python3 - <<'PY' +import base64 +import json +import os +from datetime import datetime, timezone, timedelta + +base_dir = os.environ.get("BASE_DIR") + +def decode_exp(token): + payload = token.split(".")[1] + padding = "=" * (-len(payload) % 4) + decoded = base64.urlsafe_b64decode(payload + padding) + payload_json = json.loads(decoded) + exp = payload_json.get("exp") + iat = payload_json.get("iat") + if not exp: + raise ValueError("exp missing in token payload") + return exp, iat + +paths = [p for p in os.listdir(base_dir) if p.endswith(".token")] +if not paths: + print("No access token files found") + raise SystemExit(2) +paths.sort(key=lambda p: os.path.getmtime(os.path.join(base_dir, p)), reverse=True) +path = os.path.join(base_dir, paths[0]) +with open(path, "r", encoding="utf-8") as f: + token = f.read().strip() +exp, iat = decode_exp(token) +print(path) +print("iat", datetime.fromtimestamp(iat, tz=timezone(timedelta(hours=4))).strftime("%Y-%m-%d %H:%M:%S GMT+4")) +print("exp", datetime.fromtimestamp(exp, tz=timezone(timedelta(hours=4))).strftime("%Y-%m-%d %H:%M:%S GMT+4")) +PY + ;; + *) + usage + exit 2 + ;; +esac diff --git a/devops/01_generate_yamls.sh b/devops/01_generate_yamls.sh new file mode 100755 index 0000000..38d6740 --- /dev/null +++ b/devops/01_generate_yamls.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +PROVIDER_DIR="${ROOT_DIR}/universal_rebuild" +PROFILE_DIR="${PROFILE_DIR:-}" + +if [[ "${1:-}" == "--profile" ]]; then + PROFILE_DIR="${2:-}" + shift 2 +fi + +if [[ -n "$PROFILE_DIR" ]]; then + if [[ ! -d "$PROFILE_DIR" ]]; then + echo "Error: profile directory not found: $PROFILE_DIR" >&2 + exit 2 + fi + PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env" + if [[ -f "$PROFILE_ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$PROFILE_ENV_FILE" + set +a + fi +fi + +if [[ -z "$PROFILE_DIR" ]]; then + echo "Error: --profile is required. Strict mode stores generated files per stand only." >&2 + exit 2 +fi + +resolve_root_path() { + local path_value="$1" + if [[ -z "$path_value" ]]; then + echo "" + return + fi + if [[ "$path_value" = /* ]]; then + echo "$path_value" + return + fi + echo "${ROOT_DIR}/${path_value}" +} + +SERVICES_FILE_DEFAULT="${ROOT_DIR}/devops/config/services_list.txt" +if [[ -f "${PROFILE_DIR}/services_list.txt" ]]; then + SERVICES_FILE_DEFAULT="${PROFILE_DIR}/services_list.txt" +fi +SERVICES_FILE="${SERVICES_FILE:-$SERVICES_FILE_DEFAULT}" +SERVICES_FILE="$(resolve_root_path "$SERVICES_FILE")" + +# Retry behavior (per service): attempts and delay between attempts. +REQUEST_DELAY="${REQUEST_DELAY:-0.5}" +ATTEMPTS="${ATTEMPTS:-3}" + +# Auth: either an explicit token or a token file (latest *.token if not set). +TOKEN_FILE="${TOKEN_FILE:-}" +NUBES_API_TOKEN="${NUBES_API_TOKEN:-}" +if [[ -n "$TOKEN_FILE" ]]; then + TOKEN_FILE="$(resolve_root_path "$TOKEN_FILE")" +fi + +# Назначение: +# - Берет список сервисов из services_list.txt. +# - Для каждого сервиса запрашивает единый spec через API и пишет YAML в +# ${ROOT_DIR}/universal_rebuild/resources_yaml. +# - Формат имени файла: ID_имя.yaml (например, 13_s3bucket.yaml). +# +# Источники данных: +# - Список сервисов: ${ROOT_DIR}/devops/config/services_list.txt +# - API endpoint: ${API_ENDPOINT} +# - Токен: переменная NUBES_API_TOKEN или файл .token. +# +# Формат services_list.txt: +# - В начале строки: service_id. +# - Остальное в строке — произвольная заметка для девопса (не используется). +# +# Задержки и попытки: +# - REQUEST_DELAY (секунды) между запросами и ретраями (по умолчанию 0.2). +# - ATTEMPTS — число попыток на сервис (по умолчанию 3). +# +# Логи ошибок: +# - Список неуспешных сервисов пишется в /tmp/yaml_gen_failures.txt. +# +# Токен берется из переменной NUBES_API_TOKEN. +# Если переменная пуста, берется файл .token из ${ROOT_DIR}: +# - "последний" = файл с самым новым временем изменения (ls -t | head -n 1). +if [[ -z "$NUBES_API_TOKEN" ]]; then + if [[ -z "$TOKEN_FILE" ]]; then + TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true) + fi + if [[ -n "$TOKEN_FILE" && -f "$TOKEN_FILE" ]]; then + NUBES_API_TOKEN=$(cat "$TOKEN_FILE") + fi +fi + +if [[ -z "$NUBES_API_TOKEN" ]]; then + echo "Error: NUBES_API_TOKEN or TOKEN_FILE is required" >&2 + exit 2 +fi + +if [[ ! -f "$SERVICES_FILE" ]]; then + echo "Error: services file not found: $SERVICES_FILE" >&2 + exit 2 +fi + +API_ENDPOINT="${NUBES_API_ENDPOINT:-https://deck-api.ngcloud.ru/api/v1/index.cfm}" +# Generator expects proxy-style endpoint (index.cfm?endpoint=/...). +if [[ "$API_ENDPOINT" != */index.cfm ]]; then + API_ENDPOINT="${API_ENDPOINT%/}/index.cfm" +fi + +GENERATED_DIR="${PROFILE_DIR}/generated" +YAML_OUTPUT_DIR_DEFAULT="${GENERATED_DIR}/resources_yaml" +FAILURES_FILE_DEFAULT="${GENERATED_DIR}/tmp/yaml_gen_failures.txt" + +YAML_OUTPUT_DIR="${YAML_OUTPUT_DIR:-$YAML_OUTPUT_DIR_DEFAULT}" +FAILURES_FILE="${FAILURES_FILE:-$FAILURES_FILE_DEFAULT}" +YAML_OUTPUT_DIR="$(resolve_root_path "$YAML_OUTPUT_DIR")" +FAILURES_FILE="$(resolve_root_path "$FAILURES_FILE")" + +# Убедимся, что папка есть. Полной очистки нет: обрабатываем только сервисы из списка. +mkdir -p "$YAML_OUTPUT_DIR" +mkdir -p "$(dirname "$FAILURES_FILE")" + +if [[ ! -f "${YAML_OUTPUT_DIR}/embed.go" ]]; then + cat > "${YAML_OUTPUT_DIR}/embed.go" <<'EOF' +package resources_yaml + +import "embed" + +// Files embeds unified service YAML specs for provider runtime lookup. +// +//go:embed *.yaml +var Files embed.FS +EOF +fi + +rm -f "$FAILURES_FILE" + +# Читаем список сервисов, пропуская пустые строки и комментарии. +while IFS= read -r line; do + line="${line//$'\r'/}" + if [[ -z "$line" ]]; then + continue + fi + if [[ "$line" == \#* ]]; then + continue + fi + sid=$(echo "$line" | awk '{print $1}') + svc_name="" + + if [[ -z "$sid" ]]; then + continue + fi + + if [[ ! "$sid" =~ ^[0-9]+$ ]]; then + echo "Error: invalid service id '${sid}' in ${SERVICES_FILE}" >&2 + echo "${sid} invalid_id" >> "$FAILURES_FILE" + continue + fi + + # Если имя не указано в списке — подтягиваем по API. + if [[ -z "$svc_name" ]]; then + svc_name=$(python3 - < /dev/null; then + echo "Error: output not created ${output_glob} for ${sid} (${svc_name})" >&2 + success=0 + fi + fi + + if [[ "$success" -eq 1 ]]; then + break + fi + echo "Retry ${attempt}/${ATTEMPTS} failed for ${sid} (${svc_name})" >&2 + sleep "$REQUEST_DELAY" + done + + # Фиксируем неудачные сервисы для анализа. + if [[ "$success" -ne 1 ]]; then + echo "${sid} ${svc_name}" >> "$FAILURES_FILE" + fi + + sleep "$REQUEST_DELAY" + +done < "$SERVICES_FILE" + +echo "Done. YAML files are in ${YAML_OUTPUT_DIR}" +if [[ -f "$FAILURES_FILE" ]]; then + echo "Failed services (see $FAILURES_FILE):" >&2 + cat "$FAILURES_FILE" >&2 + exit 1 +fi \ No newline at end of file diff --git a/devops/02_generate_resources_and_docs.sh b/devops/02_generate_resources_and_docs.sh new file mode 100755 index 0000000..7afcece --- /dev/null +++ b/devops/02_generate_resources_and_docs.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Generate Go resources and documentation from unified YAML. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +PROVIDER_DIR="${ROOT_DIR}/universal_rebuild" +DOCS_DIR="${ROOT_DIR}/docs/30_registry/resources" + +cd "$PROVIDER_DIR" + +# Remove old generated resources (keep core files). +rm -f "${PROVIDER_DIR}/internal/resources_gen"/*_resource.go +rm -f "${PROVIDER_DIR}/internal/resources_gen/registry.go" + +# Step 1: generate Go resources from unified YAML. +echo "Generating Go resources from unified YAML..." +go run ./tools/gen_v2 + +# Step 2: generate documentation from unified YAML (new template). +echo "Generating documentation using new template generator..." +"${ROOT_DIR}/devops/02_generate_resources_and_docs_template_v2.sh" + +echo "Done. Docs updated in ${DOCS_DIR}" diff --git a/devops/02_generate_resources_and_docs_template.sh b/devops/02_generate_resources_and_docs_template.sh new file mode 100755 index 0000000..4be385b --- /dev/null +++ b/devops/02_generate_resources_and_docs_template.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Generate docs using ClickHouse template. +# Use as docs template generator for unified resources_yaml. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +PROVIDER_DIR="${ROOT_DIR}/universal_rebuild" +RESOURCES_YAML_DIR="${PROVIDER_DIR}/resources_yaml" +DOCS_DIR="${ROOT_DIR}/docs/30_registry/resources" +SERVICES_LIST_PATH="${ROOT_DIR}/devops/config/services_list.txt" + +cd "$PROVIDER_DIR" + +go run ./tools/docs_template_gen \ + -resources "$RESOURCES_YAML_DIR" \ + -docs "$DOCS_DIR" \ + -services "$SERVICES_LIST_PATH" \ + -exclude "clickhouse" + +echo "Docs generated in ${DOCS_DIR}" diff --git a/devops/02_generate_resources_and_docs_template_v2.sh b/devops/02_generate_resources_and_docs_template_v2.sh new file mode 100755 index 0000000..c5e4f28 --- /dev/null +++ b/devops/02_generate_resources_and_docs_template_v2.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Generate docs using ClickHouse template (v2, no color markup). +# Canonical docs generator for unified resources_yaml. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +PROVIDER_DIR="${ROOT_DIR}/universal_rebuild" +RESOURCES_YAML_DIR="${PROVIDER_DIR}/resources_yaml" +DOCS_DIR="${ROOT_DIR}/docs/30_registry/resources" +SERVICES_LIST_PATH="${ROOT_DIR}/devops/config/services_list.txt" + +cd "$PROVIDER_DIR" + +go run ./tools/docs_template_gen_v2 \ + -resources "$RESOURCES_YAML_DIR" \ + -docs "$DOCS_DIR" \ + -services "$SERVICES_LIST_PATH" \ + -exclude "clickhouse" + +echo "Docs generated in ${DOCS_DIR}" diff --git a/devops/02_generate_resources_and_docs_v2.sh b/devops/02_generate_resources_and_docs_v2.sh new file mode 100755 index 0000000..201e562 --- /dev/null +++ b/devops/02_generate_resources_and_docs_v2.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Canonical resources+docs generator (template format). +# Restores instruction-style docs structure (manual/example/params_create/params_modify/outputs/ops/params landing). + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +PROVIDER_DIR="${ROOT_DIR}/universal_rebuild" +PROFILE_DIR="${PROFILE_DIR:-}" + +resolve_root_path() { + local path_value="$1" + if [[ -z "$path_value" ]]; then + echo "" + return + fi + if [[ "$path_value" = /* ]]; then + echo "$path_value" + return + fi + echo "${ROOT_DIR}/${path_value}" +} + +if [[ "${1:-}" == "--profile" ]]; then + PROFILE_DIR="${2:-}" + shift 2 +fi + +if [[ -n "$PROFILE_DIR" ]]; then + if [[ ! -d "$PROFILE_DIR" ]]; then + echo "Error: profile directory not found: $PROFILE_DIR" >&2 + exit 2 + fi + PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env" + if [[ -f "$PROFILE_ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$PROFILE_ENV_FILE" + set +a + fi +fi + +if [[ -z "$PROFILE_DIR" ]]; then + echo "Error: --profile is required. Strict mode stores generated files per stand only." >&2 + exit 2 +fi + +PROFILE_DIR="$(resolve_root_path "$PROFILE_DIR")" + +RESOURCES_YAML_DIR_DEFAULT="${PROFILE_DIR}/generated/resources_yaml" +DOCS_DIR_DEFAULT="${PROFILE_DIR}/generated/docs" +SERVICES_LIST_PATH_DEFAULT="${PROFILE_DIR}/services_list.txt" +GO_OUTPUT_DIR_DEFAULT="${PROFILE_DIR}/generated/go" + +RESOURCES_YAML_DIR="${RESOURCES_YAML_DIR:-$RESOURCES_YAML_DIR_DEFAULT}" +DOCS_DIR="${DOCS_DIR:-$DOCS_DIR_DEFAULT}" +SERVICES_LIST_PATH="${SERVICES_LIST_PATH:-$SERVICES_LIST_PATH_DEFAULT}" +GO_OUTPUT_DIR="${GO_OUTPUT_DIR:-$GO_OUTPUT_DIR_DEFAULT}" + +RESOURCES_YAML_DIR="$(resolve_root_path "$RESOURCES_YAML_DIR")" +DOCS_DIR="$(resolve_root_path "$DOCS_DIR")" +SERVICES_LIST_PATH="$(resolve_root_path "$SERVICES_LIST_PATH")" +GO_OUTPUT_DIR="$(resolve_root_path "$GO_OUTPUT_DIR")" + +if [[ ! -d "$RESOURCES_YAML_DIR" ]]; then + echo "Error: resources yaml directory not found: $RESOURCES_YAML_DIR" >&2 + exit 2 +fi + +if [[ ! -f "$SERVICES_LIST_PATH" ]]; then + echo "Error: services list not found: $SERVICES_LIST_PATH" >&2 + exit 2 +fi + +cd "$PROVIDER_DIR" + +echo "Generating resources from unified YAML specs..." +TMP_GEN_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_GEN_DIR"' EXIT + +NUBES_RESOURCES_DIR="$RESOURCES_YAML_DIR" \ +NUBES_RESOURCES_GEN_DIR="$TMP_GEN_DIR" \ +go run ./tools/gen_v2 + +mkdir -p "$GO_OUTPUT_DIR" +rm -rf "${GO_OUTPUT_DIR}"/* +cp -R "$TMP_GEN_DIR/." "$GO_OUTPUT_DIR/" + +echo "Generating docs via template generator..." +mkdir -p "$DOCS_DIR" +go run ./tools/docs_template_gen_v2 \ + -resources "$RESOURCES_YAML_DIR" \ + -docs "$DOCS_DIR" \ + -services "$SERVICES_LIST_PATH" \ + -exclude "clickhouse" + +echo "Resources and docs generated in template format." diff --git a/devops/03_build_and_upload_provider.sh b/devops/03_build_and_upload_provider.sh new file mode 100755 index 0000000..b8f88b1 --- /dev/null +++ b/devops/03_build_and_upload_provider.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +PROVIDER_MAIN="${ROOT_DIR}/universal_rebuild/main.go" +BUILD_SCRIPT="${ROOT_DIR}/devops/build-provider.sh" +PROFILE_DIR="${PROFILE_DIR:-}" + +if [[ "${1:-}" == "--profile" ]]; then + PROFILE_DIR="${2:-}" + shift 2 +fi + +if [[ -n "$PROFILE_DIR" ]]; then + if [[ ! -d "$PROFILE_DIR" ]]; then + echo "Error: profile directory not found: $PROFILE_DIR" >&2 + exit 2 + fi + PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env" + if [[ -f "$PROFILE_ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$PROFILE_ENV_FILE" + set +a + fi +fi + +if [[ -z "$PROFILE_DIR" ]]; then + echo "Error: --profile is required. Strict mode stores generated files per stand only." >&2 + exit 2 +fi + +resolve_root_path() { + local path_value="$1" + if [[ -z "$path_value" ]]; then + echo "" + return + fi + if [[ "$path_value" = /* ]]; then + echo "$path_value" + return + fi + echo "${ROOT_DIR}/${path_value}" +} + +S3CFG_REGISTRY="${S3CFG_REGISTRY:-${ROOT_DIR}/secrets/.s3cfg_registry}" +TIMEOUTS_SOURCE="${TIMEOUTS_SOURCE:-${ROOT_DIR}/devops/config/operation_timeouts.json}" +PROFILE_RESOURCES_YAML_DIR="${PROFILE_DIR}/generated/resources_yaml" +PROFILE_GENERATED_GO_DIR="${PROFILE_DIR}/generated/go" +PROFILE_BUILD_DIR="${PROFILE_DIR}/generated/provider_build" +S3CFG_REGISTRY="$(resolve_root_path "$S3CFG_REGISTRY")" +TIMEOUTS_SOURCE="$(resolve_root_path "$TIMEOUTS_SOURCE")" +PROFILE_RESOURCES_YAML_DIR="$(resolve_root_path "$PROFILE_RESOURCES_YAML_DIR")" +PROFILE_GENERATED_GO_DIR="$(resolve_root_path "$PROFILE_GENERATED_GO_DIR")" +PROFILE_BUILD_DIR="$(resolve_root_path "$PROFILE_BUILD_DIR")" + +if [[ -f "${PROFILE_DIR}/operation_timeouts.json" ]]; then + TIMEOUTS_SOURCE="${PROFILE_DIR}/operation_timeouts.json" +fi + +load_s3cfg_registry() { + local cfg="$1" + if [[ ! -f "$cfg" ]]; then + echo "Error: S3 config not found: $cfg" >&2 + exit 2 + fi + + local access_key secret_key host_base use_https endpoint + access_key=$(awk -F '=' '/^\s*access_key\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg") + secret_key=$(awk -F '=' '/^\s*secret_key\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg") + host_base=$(awk -F '=' '/^\s*host_base\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg") + use_https=$(awk -F '=' '/^\s*use_https\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print tolower($2); exit}' "$cfg") + + if [[ -z "${S3_ACCESS_KEY:-}" && -n "$access_key" ]]; then + export S3_ACCESS_KEY="$access_key" + fi + if [[ -z "${S3_SECRET_KEY:-}" && -n "$secret_key" ]]; then + export S3_SECRET_KEY="$secret_key" + fi + + if [[ -z "${S3_ENDPOINT:-}" && -n "$host_base" ]]; then + if [[ "$host_base" == http* ]]; then + endpoint="$host_base" + else + if [[ "$use_https" == "false" || "$use_https" == "0" || "$use_https" == "no" ]]; then + endpoint="http://${host_base}" + else + endpoint="https://${host_base}" + fi + fi + export S3_ENDPOINT="$endpoint" + fi + + if [[ -z "${S3_ENDPOINT:-}" || -z "${S3_ACCESS_KEY:-}" || -z "${S3_SECRET_KEY:-}" ]]; then + echo "Error: S3 credentials are not set (S3_ENDPOINT/S3_ACCESS_KEY/S3_SECRET_KEY)" >&2 + exit 2 + fi +} + +VERSION="${1:-}" +if [[ -z "$VERSION" ]]; then + VERSION="${PROVIDER_VERSION:-${RELEASE_VERSION:-}}" +fi +if [[ -z "$VERSION" ]]; then + VERSION=$(grep -E 'version string' "$PROVIDER_MAIN" | sed -E 's/.*"([0-9.]+)".*/\1/') +fi + +if [[ -z "$VERSION" ]]; then + echo "Error: unable to detect version from $PROVIDER_MAIN" >&2 + exit 2 +fi + +load_s3cfg_registry "$S3CFG_REGISTRY" + +if [[ ! -f "$TIMEOUTS_SOURCE" ]]; then + echo "Error: operation timeouts config not found: $TIMEOUTS_SOURCE" >&2 + exit 2 +fi + +if [[ ! -d "$PROFILE_RESOURCES_YAML_DIR" ]]; then + echo "Error: profile resources yaml directory not found: $PROFILE_RESOURCES_YAML_DIR" >&2 + exit 2 +fi +if ! compgen -G "${PROFILE_RESOURCES_YAML_DIR}/*.yaml" > /dev/null; then + echo "Error: no yaml specs found in profile: $PROFILE_RESOURCES_YAML_DIR" >&2 + exit 2 +fi + +if [[ ! -d "$PROFILE_GENERATED_GO_DIR" ]]; then + echo "Error: profile generated go directory not found: $PROFILE_GENERATED_GO_DIR" >&2 + exit 2 +fi +if ! compgen -G "${PROFILE_GENERATED_GO_DIR}/*.go" > /dev/null; then + echo "Error: no generated go files found in profile: $PROFILE_GENERATED_GO_DIR" >&2 + exit 2 +fi + +validate_registry_completeness() { + local resources_gen_dir="$1" + local registry_file="${resources_gen_dir}/registry.go" + if [[ ! -f "$registry_file" ]]; then + echo "Error: registry.go not found in resources_gen: $resources_gen_dir" >&2 + exit 2 + fi + + local tmp_expected tmp_registered tmp_missing + tmp_expected="$(mktemp)" + tmp_registered="$(mktemp)" + tmp_missing="$(mktemp)" + + find "$resources_gen_dir" -maxdepth 1 -type f -name '*_resource.go' -print0 \ + | xargs -0 grep -hE 'func[[:space:]]+New[[:alnum:]_]+Resource[[:space:]]*\(' \ + | sed -E 's/.*(New[[:alnum:]_]+Resource).*/\1/' \ + | sort -u > "$tmp_expected" + + grep -hEo 'New[[:alnum:]_]+Resource' "$registry_file" | sort -u > "$tmp_registered" + + comm -23 "$tmp_expected" "$tmp_registered" > "$tmp_missing" + + if [[ -s "$tmp_missing" ]]; then + echo "Error: registry.go is missing resource constructors:" >&2 + sed 's/^/ - /' "$tmp_missing" >&2 + rm -f "$tmp_expected" "$tmp_registered" "$tmp_missing" + exit 2 + fi + + rm -f "$tmp_expected" "$tmp_registered" "$tmp_missing" +} + +TMP_PROVIDER_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_PROVIDER_DIR"' EXIT + +cp -a "${ROOT_DIR}/universal_rebuild/." "$TMP_PROVIDER_DIR/" +mkdir -p "$TMP_PROVIDER_DIR/resources_yaml" +find "$TMP_PROVIDER_DIR/resources_yaml" -maxdepth 1 -type f -name '*.yaml' -delete +cp -a "${PROFILE_RESOURCES_YAML_DIR}"/*.yaml "$TMP_PROVIDER_DIR/resources_yaml/" +cp "$TIMEOUTS_SOURCE" "$TMP_PROVIDER_DIR/internal/provider/operation_timeouts.json" + +mkdir -p "$TMP_PROVIDER_DIR/internal/resources_gen" +find "$TMP_PROVIDER_DIR/internal/resources_gen" -maxdepth 1 -type f -name '*.go' -delete +cp -R "${PROFILE_GENERATED_GO_DIR}"/*.go "$TMP_PROVIDER_DIR/internal/resources_gen/" + +validate_registry_completeness "$TMP_PROVIDER_DIR/internal/resources_gen" + +mkdir -p "$PROFILE_BUILD_DIR" + +if [[ -n "${GPG_PRIVATE_KEY_PATH:-}" ]]; then + export GPG_KEY_FILE="$(resolve_root_path "$GPG_PRIVATE_KEY_PATH")" +fi + +echo "Building provider version ${VERSION}..." +PROVIDER_DIR="$TMP_PROVIDER_DIR" BUILD_DIR="$PROFILE_BUILD_DIR" "$BUILD_SCRIPT" "$VERSION" diff --git a/devops/04_build_and_publish_docs.sh b/devops/04_build_and_publish_docs.sh new file mode 100755 index 0000000..5ea8fa7 --- /dev/null +++ b/devops/04_build_and_publish_docs.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +PROVIDER_MAIN="${ROOT_DIR}/universal_rebuild/main.go" +PROFILE_DIR="${PROFILE_DIR:-}" + +normalize_api_endpoint() { + local endpoint="$1" + endpoint="${endpoint%/}" + if [[ -z "$endpoint" ]]; then + echo "https://deck-api.ngcloud.ru/api/v1/index.cfm" + return + fi + if [[ "$endpoint" == */index.cfm ]]; then + echo "$endpoint" + return + fi + echo "${endpoint}/index.cfm" +} + +if [[ "${1:-}" == "--profile" ]]; then + PROFILE_DIR="${2:-}" + shift 2 +fi + +if [[ -n "$PROFILE_DIR" ]]; then + if [[ ! -d "$PROFILE_DIR" ]]; then + echo "Error: profile directory not found: $PROFILE_DIR" >&2 + exit 2 + fi + PROFILE_ENV_FILE="${PROFILE_DIR}/profile.env" + if [[ -f "$PROFILE_ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$PROFILE_ENV_FILE" + set +a + fi +fi + +resolve_root_path() { + local path_value="$1" + if [[ -z "$path_value" ]]; then + echo "" + return + fi + if [[ "$path_value" = /* ]]; then + echo "$path_value" + return + fi + echo "${ROOT_DIR}/${path_value}" +} + +VERSION="${1:-}" +if [[ -z "$VERSION" ]]; then + VERSION="${DOCS_VERSION:-${RELEASE_VERSION:-}}" +fi +if [[ -z "$VERSION" ]]; then + VERSION=$(grep -E 'version string' "$PROVIDER_MAIN" | sed -E 's/.*"([0-9.]+)".*/\1/') +fi + +if [[ -z "$VERSION" ]]; then + echo "Error: unable to detect version from $PROVIDER_MAIN" >&2 + exit 2 +fi + +REGISTRY_HOST="${REGISTRY_HOST:-terra.k8c.ru}" +NAMESPACE="${NAMESPACE:-nubes}" +PROVIDER_NAME="${PROVIDER_NAME:-nubes}" +export REGISTRY_HOST NAMESPACE PROVIDER_NAME VERSION + +S3CFG_REGISTRY="${S3CFG_REGISTRY:-${ROOT_DIR}/secrets/.s3cfg_registry}" +S3CFG_REGISTRY="$(resolve_root_path "$S3CFG_REGISTRY")" +TMP_DOCS_DIR="" + +if [[ -n "$PROFILE_DIR" ]]; then + PROFILE_DOCS_DIR="${PROFILE_DIR}/generated/docs" + PROFILE_DOCS_DIR="$(resolve_root_path "$PROFILE_DOCS_DIR")" + if [[ -d "$PROFILE_DOCS_DIR" ]]; then + TMP_DOCS_DIR="$(mktemp -d)" + cp -a "${ROOT_DIR}/docs/." "$TMP_DOCS_DIR/" + mkdir -p "$TMP_DOCS_DIR/30_registry/resources" + cp -a "$PROFILE_DOCS_DIR/." "$TMP_DOCS_DIR/30_registry/resources/" + export MKDOCS_DOCS_DIR="$TMP_DOCS_DIR" + fi +fi + +DOCS_API_ENDPOINT="${NUBES_API_ENDPOINT:-https://deck-api.ngcloud.ru/api/v1/index.cfm}" +DOCS_API_ENDPOINT="$(normalize_api_endpoint "$DOCS_API_ENDPOINT")" + +if [[ -n "$TMP_DOCS_DIR" ]]; then + export DOCS_GUIDE_VERSION="$VERSION" + export DOCS_GUIDE_API_ENDPOINT="$DOCS_API_ENDPOINT" + export DOCS_GUIDE_FILE="$TMP_DOCS_DIR/30_registry/guides/getting-started.md" + python3 - <<'PY' +import os +import re +from pathlib import Path + +guide_path = Path(os.environ["DOCS_GUIDE_FILE"]) +if guide_path.exists(): + text = guide_path.read_text(encoding="utf-8") + text = re.sub( + r'(version\s*=\s*")([0-9.]+)("\s*#\s*Поставьте нужную вам версию провайдера)', + lambda m: f'{m.group(1)}{os.environ["DOCS_GUIDE_VERSION"]}{m.group(3)}', + text, + count=1, + ) + text = re.sub( + r'(api_endpoint\s*=\s*")([^"]+)(")', + lambda m: f'{m.group(1)}{os.environ["DOCS_GUIDE_API_ENDPOINT"]}{m.group(3)}', + text, + count=1, + ) + guide_path.write_text(text, encoding="utf-8") +PY +fi + +load_s3cfg_registry() { + local cfg="$1" + if [[ ! -f "$cfg" ]]; then + echo "Error: S3 config not found: $cfg" >&2 + exit 2 + fi + + local access_key secret_key host_base use_https endpoint + access_key=$(awk -F '=' '/^\s*access_key\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg") + secret_key=$(awk -F '=' '/^\s*secret_key\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg") + host_base=$(awk -F '=' '/^\s*host_base\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print $2; exit}' "$cfg") + use_https=$(awk -F '=' '/^\s*use_https\s*=/ {gsub(/^[ \t]+|[ \t]+$/, "", $2); print tolower($2); exit}' "$cfg") + + if [[ -z "${S3_ACCESS_KEY:-}" && -n "$access_key" ]]; then + export S3_ACCESS_KEY="$access_key" + fi + if [[ -z "${S3_SECRET_KEY:-}" && -n "$secret_key" ]]; then + export S3_SECRET_KEY="$secret_key" + fi + + if [[ -z "${S3_ENDPOINT:-}" && -n "$host_base" ]]; then + if [[ "$host_base" == http* ]]; then + endpoint="$host_base" + else + if [[ "$use_https" == "false" || "$use_https" == "0" || "$use_https" == "no" ]]; then + endpoint="http://${host_base}" + else + endpoint="https://${host_base}" + fi + fi + export S3_ENDPOINT="$endpoint" + fi + + if [[ -z "${S3_ENDPOINT:-}" || -z "${S3_ACCESS_KEY:-}" || -z "${S3_SECRET_KEY:-}" ]]; then + echo "Error: S3 credentials are not set (S3_ENDPOINT/S3_ACCESS_KEY/S3_SECRET_KEY)" >&2 + exit 2 + fi +} + +TMP_CFG="${ROOT_DIR}/.mkdocs.tmp.yml" +export ROOT_DIR +python3 - <<'PY' +import os +import re +from pathlib import Path + +registry_host = os.environ["REGISTRY_HOST"] +namespace = os.environ["NAMESPACE"] +name = os.environ["PROVIDER_NAME"] +version = os.environ["VERSION"] +root_dir = Path(os.environ["ROOT_DIR"]) + +cfg_path = root_dir / "mkdocs.yml" +text = cfg_path.read_text(encoding="utf-8") +text = re.sub( + r"^site_url:\s*.*$", + f"site_url: https://{registry_host}/docs/{namespace}/{name}/{version}/", + text, + flags=re.M, +) + +docs_dir = os.environ.get("MKDOCS_DOCS_DIR", "").strip() +if docs_dir: + if re.search(r"^docs_dir:\s*.*$", text, flags=re.M): + text = re.sub(r"^docs_dir:\s*.*$", f"docs_dir: {docs_dir}", text, flags=re.M) + else: + text += f"\ndocs_dir: {docs_dir}\n" + +(root_dir / ".mkdocs.tmp.yml").write_text(text, encoding="utf-8") +PY + +build_docs() { + local docs_build_mode + docs_build_mode="${DOCS_BUILD_MODE:-auto}" + + if [[ -n "${MKDOCS_DOCS_DIR:-}" && "$docs_build_mode" == "auto" ]]; then + docs_build_mode="local" + fi + + if [[ "$docs_build_mode" != "local" ]] && command -v docker >/dev/null 2>&1; then + local docker_err + docker_err="$(mktemp)" + if docker run --rm -v "${ROOT_DIR}:/docs" squidfunk/mkdocs-material build -f "/docs/.mkdocs.tmp.yml" -d site 2>"$docker_err"; then + rm -f "$docker_err" + return 0 + fi + + if grep -q "error while creating mount source path" "$docker_err"; then + echo "Warning: docker mount issue detected, using local mkdocs" >&2 + else + cat "$docker_err" >&2 + echo "Warning: docker build failed, trying local mkdocs" >&2 + fi + rm -f "$docker_err" + fi + + local venv_python + venv_python="${ROOT_DIR}/.venv/bin/python" + + if [[ -x "$venv_python" ]]; then + if "$venv_python" -m mkdocs --version >/dev/null 2>&1; then + (cd "$ROOT_DIR" && "$venv_python" -m mkdocs build -f "$TMP_CFG" -d site) + return 0 + fi + + echo "Warning: mkdocs module is missing in .venv, installing pinned versions" >&2 + "$venv_python" -m pip install --disable-pip-version-check -q "mkdocs==1.6.1" "mkdocs-material==9.7.3" + (cd "$ROOT_DIR" && "$venv_python" -m mkdocs build -f "$TMP_CFG" -d site) + return 0 + fi + + if command -v mkdocs >/dev/null 2>&1 && mkdocs --version >/dev/null 2>&1; then + (cd "$ROOT_DIR" && mkdocs build -f "$TMP_CFG" -d site) + return 0 + fi + + echo "Error: mkdocs is not installed or not runnable." >&2 + echo "Install once: python3 -m venv .venv && .venv/bin/python -m pip install mkdocs==1.6.1 mkdocs-material==9.7.3" >&2 + exit 2 +} + +trap 'rm -f "$TMP_CFG"; if [[ -n "$TMP_DOCS_DIR" ]]; then rm -rf "$TMP_DOCS_DIR"; fi' EXIT +build_docs + +: + +: + +load_s3cfg_registry "$S3CFG_REGISTRY" +(cd "$ROOT_DIR" && ./scripts/publish-docs.sh site "$REGISTRY_HOST" "$NAMESPACE" "$PROVIDER_NAME" "$VERSION") +echo "Docs published for ${VERSION}" \ No newline at end of file diff --git a/devops/04b_enrich_outputs_from_inventory.py b/devops/04b_enrich_outputs_from_inventory.py new file mode 100644 index 0000000..0d88564 --- /dev/null +++ b/devops/04b_enrich_outputs_from_inventory.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +""" +Дополнительный генератор: обогащает *_outputs.md реальными ключами +из output_inventory (собранного со стендов). + +Обновляет только секцию "## Реальные поля из облака" — не трогает остальное. + +Запуск (из корня репо на VM): + python3 devops/04b_enrich_outputs_from_inventory.py [--dry-run] +""" + +import json, yaml, glob, os, sys + +BASE = "/home/naeel/terra/terraform" +YAML_DIR = f"{BASE}/devops/profiles/test/generated/resources_yaml" +INV_DIR = f"{BASE}/devops/profiles/test/generated/output_inventory" +DOCS_DIR = f"{BASE}/docs/30_registry/resources" + +SECTION_MARKER = "## Реальные поля из облака" + + +def flatten(obj, prefix=""): + """Рекурсивно собирает dot-notation ключи из вложенного dict/list.""" + keys = [] + if isinstance(obj, dict): + for k, v in obj.items(): + full = f"{prefix}.{k}" if prefix else k + keys.append(full) + if isinstance(v, (dict, list)): + keys.extend(flatten(v, full)) + elif isinstance(obj, list): + for i, v in enumerate(obj): + full = f"{prefix}.{i}" if prefix else str(i) + if isinstance(v, (dict, list)): + keys.extend(flatten(v, full)) + # скалярные элементы списка не добавляем — нет смысла + return keys + + +def best_instance(instances): + """Выбираем лучший инстанс: running с данными > suspended с данными.""" + candidates = [i for i in instances if i.get("out") or i.get("params")] + if not candidates: + return None + running = [i for i in candidates if i.get("status") == "running"] + return running[-1] if running else candidates[-1] + + +def build_section(inst): + params = inst.get("params") or {} + out = inst.get("out") or {} + vault = inst.get("vault_keys") or [] + source = inst.get("source", "test") + name = inst.get("name", "?") + status = inst.get("status", "?") + + lines = [ + f"{SECTION_MARKER} ({source} / {name} / {status})\n", + "\n", + "Поля ниже получены из реальных инстансов этого сервиса в облаке.\n", + "Используйте их как готовые ключи для `state_out_flat` и `vault_secrets`.\n", + ] + + if params: + lines.append("\n### `state_params` ключи\n\n") + for k in params.keys(): + lines.append(f'- `state_params["{k}"]`\n') + + flat_out = flatten(out) + if flat_out: + lines.append("\n### `state_out_flat` ключи\n\n") + for k in flat_out: + lines.append(f'- `state_out_flat["{k}"]`\n') + + if vault: + lines.append("\n### `vault_secrets` ключи\n\n") + for k in vault: + lines.append(f'- `vault_secrets["{k}"]`\n') + + return "".join(lines) + + +def main(): + dry_run = "--dry-run" in sys.argv + + # 1. service_id → name (из YAML) + svc_to_name = {} + for f in glob.glob(f"{YAML_DIR}/*.yaml"): + try: + d = yaml.safe_load(open(f)) + sid = d.get("service_id") + name = d.get("name") or d.get("service_short_name") + if sid and name: + svc_to_name[sid] = name + except Exception: + pass + + updated = 0 + skipped = 0 + + for inv_file in sorted(glob.glob(f"{INV_DIR}/svc_*.json")): + if "index" in inv_file: + continue + + inv = json.load(open(inv_file)) + svc_id = inv.get("serviceId") + inst = best_instance(inv.get("instances", [])) + + if not inst: + print(f" skip svc={svc_id}: нет данных") + skipped += 1 + continue + + name = svc_to_name.get(svc_id) + if not name: + print(f" skip svc={svc_id}: нет YAML маппинга") + skipped += 1 + continue + + out_path = f"{DOCS_DIR}/{name}_outputs.md" + if not os.path.exists(out_path): + print(f" skip svc={svc_id} ({name}): нет файла {name}_outputs.md") + skipped += 1 + continue + + new_section = build_section(inst) + content = open(out_path).read() + + if SECTION_MARKER in content: + idx = content.index(SECTION_MARKER) + new_content = content[:idx] + new_section + else: + new_content = content.rstrip("\n") + "\n\n" + new_section + + if dry_run: + print(f" dry-run svc={svc_id} ({name}): {out_path}") + print(f" params={list((inst.get('params') or {}).keys())}") + print(f" out_flat_count={len(flatten(inst.get('out') or {}))}") + print(f" vault={inst.get('vault_keys', [])}") + else: + with open(out_path, "w") as f: + f.write(new_content) + print(f" updated svc={svc_id} ({name})") + + updated += 1 + + print(f"\nДone: {updated} updated, {skipped} skipped") + + +if __name__ == "__main__": + main() diff --git a/devops/10_yaml_stability_run.sh b/devops/10_yaml_stability_run.sh new file mode 100755 index 0000000..a427156 --- /dev/null +++ b/devops/10_yaml_stability_run.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +SCRIPT="${ROOT_DIR}/devops/01_generate_yamls.sh" +TOKEN_FILE="${TOKEN_FILE:-}" +REQUEST_DELAY="${REQUEST_DELAY:-0.2}" +ATTEMPTS="${ATTEMPTS:-3}" +RUNS="${RUNS:-10}" +LOG="${LOG:-/tmp/yaml_gen_runs.log}" + +export TOKEN_FILE + +if [[ -z "$TOKEN_FILE" ]]; then + TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true) +fi + +if [[ -z "$TOKEN_FILE" || ! -f "$TOKEN_FILE" ]]; then + echo "Error: token file not found in ${ROOT_DIR}" >&2 + exit 2 +fi + +: > "$LOG" + +echo "Runs: $RUNS" | tee -a "$LOG" +echo "Request delay: $REQUEST_DELAY" | tee -a "$LOG" +echo "Attempts per service: $ATTEMPTS" | tee -a "$LOG" +echo "Token file: $TOKEN_FILE" | tee -a "$LOG" + +echo "" | tee -a "$LOG" + +for i in $(seq 1 "$RUNS"); do + start=$(date -u +%Y-%m-%dT%H:%M:%SZ) + echo "RUN $i start $start" | tee -a "$LOG" + REQUEST_DELAY="$REQUEST_DELAY" ATTEMPTS="$ATTEMPTS" "$SCRIPT" >> "$LOG" 2>&1 + rc=$? + end=$(date -u +%Y-%m-%dT%H:%M:%SZ) + echo "RUN $i end $end rc=$rc" | tee -a "$LOG" + echo "" | tee -a "$LOG" + sleep 0.5 +done + +echo "Log: $LOG" | tee -a "$LOG" diff --git a/devops/11_yaml_stability_run_latest.sh b/devops/11_yaml_stability_run_latest.sh new file mode 100755 index 0000000..a0fce12 --- /dev/null +++ b/devops/11_yaml_stability_run_latest.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +TOKEN_FILE="${TOKEN_FILE:-}" + +if [[ -z "$TOKEN_FILE" ]]; then + TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true) +fi + +if [[ -z "$TOKEN_FILE" || ! -f "$TOKEN_FILE" ]]; then + echo "Error: token file not found in ${ROOT_DIR}" >&2 + exit 2 +fi + +export TOKEN_FILE +exec "${ROOT_DIR}/devops/10_yaml_stability_run.sh" diff --git a/devops/12_generate_yamls_latest.sh b/devops/12_generate_yamls_latest.sh new file mode 100755 index 0000000..95ccbbd --- /dev/null +++ b/devops/12_generate_yamls_latest.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +TOKEN_FILE="${TOKEN_FILE:-}" + +if [[ -z "$TOKEN_FILE" ]]; then + TOKEN_FILE=$(ls -t "${ROOT_DIR}"/*.token 2>/dev/null | head -n 1 || true) +fi + +if [[ -z "$TOKEN_FILE" || ! -f "$TOKEN_FILE" ]]; then + echo "Error: token file not found in ${ROOT_DIR}" >&2 + exit 2 +fi + +export TOKEN_FILE +exec "${ROOT_DIR}/devops/01_generate_yamls.sh" diff --git a/devops/13_generate_yamls_clean.sh b/devops/13_generate_yamls_clean.sh new file mode 100755 index 0000000..125b06f --- /dev/null +++ b/devops/13_generate_yamls_clean.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +PROVIDER_DIR="${ROOT_DIR}/universal_rebuild" + +if [[ ! -d "${PROVIDER_DIR}/resources_yaml" ]]; then + echo "Error: resources_yaml not found: ${PROVIDER_DIR}/resources_yaml" >&2 + exit 2 +fi + +# Start from a clean slate to avoid stale YAMLs. +rm -f "${PROVIDER_DIR}/resources_yaml"/*.yaml + +exec "${ROOT_DIR}/devops/12_generate_yamls_latest.sh" diff --git a/devops/ARCHITECTURE.md b/devops/ARCHITECTURE.md new file mode 100644 index 0000000..2d84f0c --- /dev/null +++ b/devops/ARCHITECTURE.md @@ -0,0 +1,131 @@ +# Provider Architecture (Source of Truth) + +This document defines the project-wide architecture and rules for generation, +provider behavior, and documentation. It is the single source of truth for +implementation and DevOps workflow. + +## Core Principles + +1) YAML per service is generated ONLY from API data. +2) The provider core is universal and must not contain service-specific logic. +3) Service-specific Go code is fully generated from YAML. No manual edits. +4) Documentation is generated from the same YAML. +5) Build artifacts for 3 OS targets are published to the registry, and docs are + published to the website. + +## Service Selection + +- The inclusion list is defined by: `devops/config/services_list.txt` (repo-relative path) +- Each line starts with service_id, followed by service name/alias. +- Operation timeouts source is defined by: `devops/config/operation_timeouts.json`. + +## Unified YAML (Per Service) + +One YAML file per service. This is the only input for: +- Provider code generation +- Documentation generation +- Validation rules + +Required top-level fields: +- name +- service_id +- service_display_name +- service_short_name +- lifecycle +- outputs +- operations +- service_man + +### Operations: Kinds and Rules + +Each operation has a kind: +- kind: instance (CRUD for the service instance, plus suspend/resume) +- kind: subresource (CRUD for objects inside the service, e.g. users/databases) +- kind: action (one-shot operations, e.g. restart/recovery/redeploy) + +### Idempotency Rules + +- instance CRUD is idempotent via standard Terraform behavior. +- subresource CRUD is idempotent by resource identity. +- action operations are idempotent via a trigger field (e.g. run_id/nonce). + If the trigger does not change, the action is not re-run. + +### Lifecycle Behavior + +- For `instance` resources with `suspend`/`resume`, use explicit flags: + `adopt_existing_on_create` (default `false`) and `suspend_on_destroy` (default `true`). +- Apply decision matrix for suspend-capable services: + - cloud status `missing` or `deleted` -> `create` + - `suspend` + `adopt_existing_on_create=true` + key params match -> `resume` + `adopt` + - `suspend` + `adopt_existing_on_create=false` -> error (explicitly require flag for resume/adopt) + - `suspend` + key params mismatch -> error + - `running` + `adopt_existing_on_create=true` -> adopt/import behavior + - `running` + `adopt_existing_on_create=false` -> error + - `not created` -> error, no auto-adopt/create + - `creating`/`pending`/`failed` -> error +- Conflict diagnostics requirement: + - When `resource_name` already exists and `adopt_existing_on_create=false`, diagnostics must explicitly offer two choices: + 1) change `resource_name` to create a new resource; + 2) import/adopt existing one by setting `adopt_existing_on_create=true` and re-running `apply`. +- Destroy behavior for suspend-capable services: + - `suspend_on_destroy=true` -> call `suspend` + - `suspend_on_destroy=false` -> remove from Terraform state only (no API call) + +### Diagnostics Format + +- Lifecycle diagnostics for plan/apply must be multiline and human-readable. +- Include decision reason and controlling flag in message body. +- Print details as separate lines: `resource_name`, `service_id`, `instance_uid`, + `status`, `status_raw`, `operation_pending`, `operation_in_progress`. + +## Provider Model + +- Core is universal: no service-specific logic inside the core. +- Generated service resources contain only schema/params and references. + +### Subresource Resources + +Subresource operations are exposed as standard resources. +Example mapping: +- create_user/delete_user/modify_user => nubes__user +- create_database/delete_database => nubes__database + +Example HCL: + +resource "nubes_postgres_user" "user1" { + postgres_id = nubes_postgres.db.id + username = "app_user" + role = "app_user" +} + +### Action Resources + +Action operations are exposed as action resources with a trigger field. +Example HCL: + +resource "nubes_postgres_restart" "restart1" { + postgres_id = nubes_postgres.db.id + run_id = "2026-02-20-001" +} + +## Documentation Model + +From the unified YAML, generate: +- Resource page: CRUD params, outputs, lifecycle defaults, operations summary +- MAN page: service_man + parameter man blocks +- Resources index: each resource links to its page and its MAN page + +## Pipeline Overview (DevOps) + +1) Generate unified YAML from API for services_list.txt. +2) Generate provider code from YAML. +3) Generate documentation from YAML. +4) Build provider for linux/windows/darwin. +5) Upload provider artifacts to registry. +6) Build and publish docs to site. + +## Non-Negotiable Rules + +- No manual edits to generated YAML or generated Go code. +- Any change must come from API or generator logic updates. +- The generator must enforce these rules and fail fast on drift. diff --git a/devops/README.md b/devops/README.md new file mode 100644 index 0000000..824da4e --- /dev/null +++ b/devops/README.md @@ -0,0 +1,258 @@ +# DevOps Runbook: Provider Build Pipeline + +This folder describes the full pipeline from API discovery to provider + docs publication. + +## Read This First + +1) Start here: `devops/README.md` (this file). +1a) Architecture: `devops/ARCHITECTURE.md`. +1b) Mandatory suspend lifecycle policy: `docs/60_strategy/provider_philosophy.md` (sections 7-9). +2) Service list: `devops/config/services_list.txt`. +2a) Operation timeouts config: `devops/config/operation_timeouts.json`. +3) Unified YAML generator: `universal_rebuild/tools/service_spec_gen`. +4) Go resource generator: `universal_rebuild/tools/gen_v2`. +5) Docs generator: `devops/02_generate_resources_and_docs_v2.sh`. +6) Build + publish scripts: `devops/03_build_and_upload_provider.sh`, `devops/04_build_and_publish_docs.sh`. + +## Overview (end-to-end) + +1) Fetch all service metadata from API +2) Generate unified YAML specs from API +3) Generate Go resources + documentation files from unified YAML +4) Build provider binaries for 3 OS targets and upload to Registry +5) Build and publish documentation site + +## Generator code locations + +- Unified YAML from API: `universal_rebuild/tools/service_spec_gen` +- Go resources from YAML: `universal_rebuild/tools/gen_v2` +- Docs generator: `devops/02_generate_resources_and_docs_v2.sh` + +## Prerequisites + +- Go 1.22+ +- `python3` +- `gpg` +- `mc` (MinIO/S3 client) +- Docker (for mkdocs build) + +## Shared settings + +S3 environment: +- `S3_ENDPOINT` (example: `https://s3.msk-1.ngcloud.ru`) +- `S3_ACCESS_KEY` +- `S3_SECRET_KEY` + +Tip: the local S3 config lives at `secrets/.s3cfg_registry`. + +Operation timeouts config: +- Source of truth: `devops/config/operation_timeouts.json` +- During provider build, this file is copied into embedded provider config by `devops/03_build_and_upload_provider.sh`. + +Provider naming defaults: +- `REGISTRY_HOSTNAME`: `terra.k8c.ru` +- `NAMESPACE`: `nubes` +- `NAME`: `nubes` + +## Stand profiles (TEST/PROD/DEV) + +To avoid overwrite between stands, use per-stand profiles: + +- `devops/profiles/test` +- `devops/profiles/prod` +- `devops/profiles/dev` + +Each profile contains: + +- `profile.env` (API URL, token file, versions, S3/GPG paths) +- `services_list.txt` +- `operation_timeouts.json` +- `generated/` (`resources_yaml`, `go`, `docs`, `json`, `tmp`) + +Run scripts with profile path: + +```bash +./devops/01_generate_yamls.sh --profile devops/profiles/test +./devops/02_generate_resources_and_docs_v2.sh --profile devops/profiles/test +./devops/03_build_and_upload_provider.sh --profile devops/profiles/test +./devops/04_build_and_publish_docs.sh --profile devops/profiles/test +``` + +`--profile` is required for generation/build scripts in strict mode. +Generated artifacts are stored only in `devops/profiles//generated/*`. +`universal_rebuild` is treated as code-only source directory. + +## Step 1: Generate unified YAML from API + +Script: `01_generate_yamls.sh` + +Input list of services: +- `config/services_list.txt` (service_id only) + +Token options: +- `TOKEN_FILE=/HH-MM-SS.token`, or +- `NUBES_API_TOKEN` directly + +Example: +```bash +export TOKEN_FILE=/08-33-41.token +./devops/01_generate_yamls.sh +``` + +Outputs: +- YAML files in `universal_rebuild/resources_yaml` (file names use `ID_name.yaml`) + +Note: +- Every run regenerates all YAML specs from the API for the services listed in `services_list.txt`. +- This overwrites previous YAMLs and ensures resources/docs match the current API for that list. +- Before generation, the scripts clean old YAML and generated Go files, so each run starts from a clean slate. + +## Step 2: Generate Go resources and docs + +Script: `02_generate_resources_and_docs_v2.sh` + +Example: +```bash +./devops/02_generate_resources_and_docs_v2.sh +``` + +Outputs: +- Go files in `universal_rebuild/internal/resources_gen` +- Docs in `docs/30_registry/resources` + +## Step 3: Build and upload provider + +Script: `03_build_and_upload_provider.sh` + +Uses `registry-server-build/build-provider.sh` and signs with: +- `secrets/private_key.asc` (ignored by git) + +Example: +```bash +export S3_ENDPOINT=https://s3.msk-1.ngcloud.ru +export S3_ACCESS_KEY=... +export S3_SECRET_KEY=... +./devops/03_build_and_upload_provider.sh 2.0.2 +``` + +## Step 4: Build and publish docs + +Script: `04_build_and_publish_docs.sh` + +Example: +```bash +export S3_ENDPOINT=https://s3.msk-1.ngcloud.ru +export S3_ACCESS_KEY=... +export S3_SECRET_KEY=... +./devops/04_build_and_publish_docs.sh 2.0.2 +``` + +## Notes + +- The GPG private key must remain stable across releases. Do not regenerate per build. +- If the key is regenerated, the registry server must be updated to serve the new public key. +- Terraform will fail with `authentication signature from unknown issuer` if the registry public key does not match the signing key. +- `config/services_list.txt` is the source of truth for which services are generated. +- If the provider version changes, update `universal_rebuild/main.go`. + +## Quickstart (fresh run) + +```bash +# 0) Prepare token +export TOKEN_FILE=/HH-MM-SS.token + +# 1) Generate YAML specs from API +./devops/01_generate_yamls.sh + +# 2) Generate Go resources + docs +./devops/02_generate_resources_and_docs_v2.sh + +# 3) Build + upload provider +./devops/03_build_and_upload_provider.sh 2.0.2 + +# 4) Build + publish docs +./devops/04_build_and_publish_docs.sh 2.0.2 +``` + +## Troubleshooting checklist + +- If YAML generation fails: check token, API endpoint, and `/tmp/yaml_gen_failures.txt`. +- If docs are missing: run `02_generate_resources_and_docs_v2.sh`. +- If registry rejects artifacts: verify GPG key and public key in registry server. + +## One-time GPG bootstrap (do this once, keep the key stable) + +1) Generate and export keys (no passphrase): +```bash +GPG_DIR=/secrets +GNUPGHOME=$(mktemp -d) +cat > /tmp/gpg_batch <<'EOF' +%no-protection +Key-Type: RSA +Key-Length: 4096 +Subkey-Type: RSA +Subkey-Length: 4096 +Name-Real: tazet@narod.ru +Name-Email: tazet@narod.ru +Expire-Date: 0 +EOF +gpg --batch --homedir "$GNUPGHOME" --gen-key /tmp/gpg_batch +gpg --batch --homedir "$GNUPGHOME" --armor --export-secret-keys > "$GPG_DIR/private_key.asc" +gpg --batch --homedir "$GNUPGHOME" --armor --export > "$GPG_DIR/public_key.asc" +rm -rf "$GNUPGHOME" /tmp/gpg_batch +``` + +2) Update registry server public key (ASCII Armor) in: +- `registry-server-build/main.go` +- `operator/cmd/registry/main.go` + +3) Rebuild and redeploy the registry server (see `docs/50_history/00_system_mechanics.md`). + +4) Build and upload provider artifacts as usual. + +## Token and self-run shortcuts + +Token capture (JSON from F12): +```bash +cat token.json | ./devops/00_token_manager.sh save - +``` + +Use latest token automatically: +```bash +./devops/12_generate_yamls_latest.sh +``` + +10-run stability test (latest token): +```bash +REQUEST_DELAY=0.3 ATTEMPTS=3 ./devops/11_yaml_stability_run_latest.sh +``` + +Log output: +```bash +tail -n 200 /tmp/yaml_gen_runs.log +``` + +## Legacy (deprecated) + +- Ops YAML and ops-doc generators were removed after the unified YAML migration. + +## Docs build policy (updated) + +- `mkdocs`/`mkdocs-material` are **not** installed on every docs generation run. +- Preferred path: build docs in Docker (`squidfunk/mkdocs-material`) via `04_build_and_publish_docs.sh`. +- Fallback path: local `mkdocs` is used only if it is already installed. +- If local `mkdocs` is missing, install once in persistent virtualenv and reuse. + +One-time setup example: + +```bash +python3 -m venv .venv +.venv/bin/pip install mkdocs mkdocs-material +``` + +Then run docs publish normally: + +```bash +./devops/04_build_and_publish_docs.sh +``` + diff --git a/devops/build-provider.sh b/devops/build-provider.sh new file mode 100755 index 0000000..f8d2a5e --- /dev/null +++ b/devops/build-provider.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Build and upload the Terraform provider to the registry S3 bucket. +# +# GPG signing key requirements: +# - A private key is required for signing SHA256SUMS. +# - The private key must be available at: +# ${ROOT_DIR}/secrets/private_key.asc +# - The private key is intentionally ignored by git. +# +# If the key is missing, generate it with a no-passphrase batch run, then +# export it to the expected location. Example (email only, no passphrase): +# GPG_DIR=${ROOT_DIR}/secrets +# GNUPGHOME=$(mktemp -d) +# cat > /tmp/gpg_batch <<'EOF' +# %no-protection +# Key-Type: RSA +# Key-Length: 3072 +# Subkey-Type: RSA +# Subkey-Length: 3072 +# Name-Real: tazet@narod.ru +# Name-Email: tazet@narod.ru +# Expire-Date: 0 +# EOF +# gpg --batch --homedir "$GNUPGHOME" --gen-key /tmp/gpg_batch +# gpg --batch --homedir "$GNUPGHOME" --armor --export-secret-keys > "$GPG_DIR/private_key.asc" +# gpg --batch --homedir "$GNUPGHOME" --armor --export > "$GPG_DIR/public_key.asc" +# rm -rf "$GNUPGHOME" /tmp/gpg_batch +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="${ROOT_DIR:-$(cd "${SCRIPT_DIR}/.." && pwd)}" +PROVIDER_DIR="${PROVIDER_DIR:-${ROOT_DIR}/universal_rebuild}" +BUILD_DIR="${BUILD_DIR:-${ROOT_DIR}/artifacts/provider_build}" +GPG_KEY_FILE="${GPG_KEY_FILE:-${ROOT_DIR}/secrets/private_key.asc}" + +VERSION="${1:-2.0.2}" +REGISTRY_HOSTNAME="${REGISTRY_HOSTNAME:-terra.k8c.ru}" +NAMESPACE="${NAMESPACE:-nubes}" +PROVIDER_NAME="${PROVIDER_NAME:-nubes}" +S3_BUCKET="${S3_BUCKET:-terraform-registry}" + +S3_ENDPOINT="${S3_ENDPOINT:-}" +S3_ACCESS_KEY="${S3_ACCESS_KEY:-}" +S3_SECRET_KEY="${S3_SECRET_KEY:-}" + +if [[ -z "$S3_ENDPOINT" || -z "$S3_ACCESS_KEY" || -z "$S3_SECRET_KEY" ]]; then + echo "Error: S3_ENDPOINT/S3_ACCESS_KEY/S3_SECRET_KEY must be set" >&2 + echo "Hint: see ${ROOT_DIR}/secrets/.s3cfg_registry" >&2 + exit 2 +fi + +if [[ ! -f "$GPG_KEY_FILE" ]]; then + echo "Error: GPG key file not found at $GPG_KEY_FILE" >&2 + exit 2 +fi + +if [[ ! -d "$PROVIDER_DIR" ]]; then + echo "Error: provider source dir not found: $PROVIDER_DIR" >&2 + exit 2 +fi + +s3_endpoint_url="$S3_ENDPOINT" +if [[ "$s3_endpoint_url" != http* ]]; then + s3_endpoint_url="https://${s3_endpoint_url}" +fi + +rm -rf "$BUILD_DIR" +mkdir -p "$BUILD_DIR" + +build_and_zip() { + local os="$1" + local arch="$2" + local binary="terraform-provider-nubes_v${VERSION}" + + if [[ "$os" == "windows" ]]; then + binary+=".exe" + fi + + echo "Building for ${os}/${arch}..." + (cd "$PROVIDER_DIR" && CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" go build -o "${BUILD_DIR}/${binary}" .) + + (cd "$BUILD_DIR" && python3 -m zipfile -c "terraform-provider-nubes_${VERSION}_${os}_${arch}.zip" "$binary") + rm -f "${BUILD_DIR:?}/${binary}" +} + +build_and_zip linux amd64 +build_and_zip windows amd64 +build_and_zip darwin amd64 + +( cd "$BUILD_DIR" && sha256sum *.zip > "terraform-provider-nubes_${VERSION}_SHA256SUMS" ) + +GNUPGHOME="$(mktemp -d)" +export GNUPGHOME +trap 'rm -rf "$GNUPGHOME"' EXIT + +gpg --batch --import "$GPG_KEY_FILE" >/dev/null +key_id=$(gpg --list-keys --with-colons | awk -F: '/^pub/{print $5; exit}') +if [[ -z "$key_id" ]]; then + echo "Error: failed to read key id from $GPG_KEY_FILE" >&2 + exit 2 +fi + +gpg --batch --yes --detach-sign --default-key "$key_id" \ + --output "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS.sig" \ + "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS" + +mc alias set registry "$s3_endpoint_url" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" --api S3v4 +S3_PATH="registry/${S3_BUCKET}/${REGISTRY_HOSTNAME}/${NAMESPACE}/${PROVIDER_NAME}/${VERSION}/" + +echo "Uploading to: ${S3_PATH}" +mc cp "${BUILD_DIR}"/*.zip "$S3_PATH" +mc cp "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS" "$S3_PATH" +mc cp "${BUILD_DIR}/terraform-provider-nubes_${VERSION}_SHA256SUMS.sig" "$S3_PATH" + +echo "Done. Version ${VERSION} uploaded." diff --git a/devops/ci/pipeline.yaml b/devops/ci/pipeline.yaml new file mode 100644 index 0000000..a683ff6 --- /dev/null +++ b/devops/ci/pipeline.yaml @@ -0,0 +1,64 @@ +# GitLab CI - pipeline для сборки и публикации Terraform Provider Registry +stages: + - test + - build + - sign + - publish + +variables: + HARBOR_HOST: "pearlharbor.registryk8s.services.ngcloud.ru" + S3_BUCKET: "terraform-registry" + PROVIDER_NAME: "nubes" + PROVIDER_NAMESPACE: "nubes" + +test: + stage: test + image: golang:1.24 + script: + - cd universal_rebuild + - go test ./... + - go vet ./... + +build-provider: + stage: build + image: golang:1.24 + script: + - cd universal_rebuild + - GOOS=linux GOARCH=amd64 go build -o bin/terraform-provider-${PROVIDER_NAME}_linux_amd64 + - GOOS=darwin GOARCH=amd64 go build -o bin/terraform-provider-${PROVIDER_NAME}_darwin_amd64 + - GOOS=windows GOARCH=amd64 go build -o bin/terraform-provider-${PROVIDER_NAME}_windows_amd64.exe + artifacts: + paths: [universal_rebuild/bin/] + +build-images: + stage: build + script: + - docker build -t ${HARBOR_HOST}/terraform/registry-server:${CI_COMMIT_TAG} registry-server-build/ + - docker build -t ${HARBOR_HOST}/terraform/registry-operator:${CI_COMMIT_TAG} operator/ + - docker push ${HARBOR_HOST}/terraform/registry-server:${CI_COMMIT_TAG} + - docker push ${HARBOR_HOST}/terraform/registry-operator:${CI_COMMIT_TAG} + only: + - tags + +sign: + stage: sign + image: golang:1.24 + script: + - cd universal_rebuild/bin + - sha256sum terraform-provider-* > SHA256SUMS + - gpg --import $GPG_PRIVATE_KEY + - gpg --detach-sign SHA256SUMS + artifacts: + paths: + - universal_rebuild/bin/SHA256SUMS + - universal_rebuild/bin/SHA256SUMS.gpg + +publish-to-s3: + stage: publish + image: alpine:3.19 + before_script: + - apk add --no-cache s3cmd + script: + - s3cmd put universal_rebuild/bin/* s3://${S3_BUCKET}/terraform-providers/ + only: + - tags diff --git a/devops/config/operation_timeouts.json b/devops/config/operation_timeouts.json new file mode 100644 index 0000000..cbf0baa --- /dev/null +++ b/devops/config/operation_timeouts.json @@ -0,0 +1,17 @@ +{ + "default_timeout": "10m", + "idle_timeout": "10m", + "operations": { + "create": "10m", + "modify": "7m", + "resume": "7m", + "suspend": "7m", + "create_user": "7m", + "delete_user": "6m", + "create_database": "7m", + "delete_database": "6m" + }, + "overrides": { + "services.90.create": "7m" + } +} diff --git a/devops/config/services_list.txt b/devops/config/services_list.txt new file mode 100644 index 0000000..5706525 --- /dev/null +++ b/devops/config/services_list.txt @@ -0,0 +1,46 @@ +# service_id service_name # RU description +# do not delete lines; to exclude a service, comment it with # +1 dummy # Болванка для тестов +2 template # Болванка для тестирования +12 s3 # Масштабируемое хранилище данных +13 s3bucket # Бакет в сервисе S3 +19 vcOrg # Организация в Cloud Director +20 vcOrgSaas # Без интеграции Keycloak +21 vc_vdc # Выделение вычислительных ресурсов +22 vc_nsxt # Обеспечение сетевой доступности в тенанте +23 vc_vm # Виртуальная машина без каких-либо особенностей.
Данная услуга запускает vc_ip для выдачи белого адреса и подключения к ВМ +# 24 DEPRECATED Правила маршрутизации для VM (vc_nat) # excluded from active generation +25 vcexternalip # Резервирование IP адреса +26 vapp # Группировка виртуальных машин +27 vc_vm_v2 # Виртуальная машина с возможностью настройки Cloud Init
Можно развернуть в услуге несколько ВМ
В зависимости от настроек можно выдавать на ВМ vip + dns +28 vc_vm_v3 # Создание виртуальной машины +29 vcVdcGroup # Обеспечение сетевой доступности между датацентрами +32 vmpostgre # Сервис настройки PostgreSQL для виртуальной машины.
Требует наличия ВМ +50 nextcloud # Платформа совместной работы +81 superset # Сервер отчетов +82 harbor # Реестр для хранения образов контейнеров +88 k8sZitiController # Контроллер управления Client to service & service to service Zero trust network
+89 flask # Flask +90 postgres # Управляемая база данных PostgreSQL +91 redis # Управляемая база данных Redis +92 mongodb # Управляемая база данных MongoDB +93 rabbitmq # Сервер очередей +94 lucee # Cервер приложений +95 nodejs # Cервер приложений +96 pgadmin # Административный интерфейс для PostgreSQL +97 nodered # Low-code платформа для IoT +98 http # Простой HTTP контейнер +99 gitea # Сервис контроля версий +100 openwhisk # Добавление и удаления пользователя (учетной записи, тенанта) для работы с Openwhisk +110 dnszone # Хостинг DNS зоны +111 dnsrecord # DNS запись в созданной зоне +112 tenant # Организация доступов к мониторингу +113 vcComplex # Создание готовой иерархии ресурсов +114 GiteaComplex # Комплексная услуга из ["gittea", "postgres"] +115 mariadb # Управляемая база данных MySQL +116 kafka # Сервис создания кластера Apache Kafka +117 nifi # ETL-система +119 Akhq # Web интерфейс Akhq для работы с Apache Kafka +120 clickhouse # Управляемая база данных ClickHouse +149 valoTenant # Корпоративная система обмена и синхронизации файлов +150 k8sSthutrvalCluster # Платформа управления K8s
diff --git a/devops/profiles/dev/operation_timeouts.json b/devops/profiles/dev/operation_timeouts.json new file mode 100644 index 0000000..cbf0baa --- /dev/null +++ b/devops/profiles/dev/operation_timeouts.json @@ -0,0 +1,17 @@ +{ + "default_timeout": "10m", + "idle_timeout": "10m", + "operations": { + "create": "10m", + "modify": "7m", + "resume": "7m", + "suspend": "7m", + "create_user": "7m", + "delete_user": "6m", + "create_database": "7m", + "delete_database": "6m" + }, + "overrides": { + "services.90.create": "7m" + } +} diff --git a/devops/profiles/dev/profile.env b/devops/profiles/dev/profile.env new file mode 100644 index 0000000..533ae11 --- /dev/null +++ b/devops/profiles/dev/profile.env @@ -0,0 +1,20 @@ +# Stand profile: DEV +NUBES_API_ENDPOINT="https://deck-api-dev.ngcloud.ru/api/v1" +TOKEN_FILE="secrets/dev.token" + +# Release versions +RELEASE_VERSION="3.0.1" +PROVIDER_VERSION="3.0.1" +DOCS_VERSION="3.0.1" + +# Registry/S3 settings +REGISTRY_HOST="terra.k8c.ru" +REGISTRY_HOSTNAME="terra.k8c.ru" +NAMESPACE="nubes" +PROVIDER_NAME="nubes" +S3_BUCKET="terraform-registry" +S3CFG_REGISTRY="secrets/.s3cfg_registry" + +# Signing keys +GPG_PRIVATE_KEY_PATH="secrets/private_key.asc" +GPG_PUBLIC_KEY_PATH="secrets/public_key.asc" diff --git a/devops/profiles/dev/services_list.txt b/devops/profiles/dev/services_list.txt new file mode 100644 index 0000000..12efb00 --- /dev/null +++ b/devops/profiles/dev/services_list.txt @@ -0,0 +1,49 @@ +# service_id service_name # RU description +# DEV stand — generated from dev API 2026-03-26 +# do not delete lines; to exclude a service, comment it with # +1 dummy # Болванка +2 template # Темплейт k8s +12 s3 # S3 Object Storage +13 s3bucket # S3 бакет +19 vcOrg # Организация в Cloud Director +# 20 vcOrgSaas # DEPRECATED — Организация без Keycloak (isProductionReady=false) +21 vc_vdc # Виртуальный датацентр (vDC) +22 vc_nsxt # Сетевой шлюз периметра (Edge) +# 23 vc_vm # DEPRECATED — VM в Cloud Director старый формат (isProductionReady=false) +# 24 vcNat # DEPRECATED — Правила маршрутизации VM (isProductionReady=false) +25 vcexternalip # Публичные IP адреса +26 vapp # Виртуальный каталог ВМ (vApp) +# 27 vc_vm_v2 # VM в Cloud Director v2 (isProductionReady=false) +28 vc_vm_v3 # Виртуальная машина +29 vcVdcGroup # Группа датацентров +# 32 vmpostgre # vc_vm_postgresql_std (isProductionReady=false) +50 nextcloud # Nextcloud +81 superset # Apache Superset +82 harbor # Container Registry +88 k8sZitiController # k8sZitiController +89 flask # Web-сервер с фреймворком Flask +90 postgres # PostgreSQL +91 redis # Redis +92 mongodb # MongoDB +93 rabbitmq # RabbitMQ +94 lucee # Lucee +95 nodejs # NodeJS +96 pgadmin # pgAdmin +97 nodered # NodeRed +98 http # Простой HTTP контейнер +99 gitea # Gitea +# 100 openwhisk # Serverless Openwhisk (isProductionReady=false) +109 zonesV2 # Управление днс-зонами и записями +110 dnszone # DNS зона +111 dnsrecord # DNS запись +# 112 tenant # Тенант в Grafana (isProductionReady=false) +# 113 vcComplex # Быстрый старт (isProductionReady=false) +115 mariadb # Mariadb +116 kafka # ApacheKafka +119 Akhq # Akhq +120 clickhouse # ClickHouse +149 valoTenant # VALO Cloud +150 k8sSthutrvalCluster # Kubernetes кластер Штурвал +151 k8sOpenbao # Vault +# 152 template # Темплейт k8s-2-clone (isProductionReady=false) +153 nifi # Nifi diff --git a/devops/profiles/prod/operation_timeouts.json b/devops/profiles/prod/operation_timeouts.json new file mode 100644 index 0000000..cbf0baa --- /dev/null +++ b/devops/profiles/prod/operation_timeouts.json @@ -0,0 +1,17 @@ +{ + "default_timeout": "10m", + "idle_timeout": "10m", + "operations": { + "create": "10m", + "modify": "7m", + "resume": "7m", + "suspend": "7m", + "create_user": "7m", + "delete_user": "6m", + "create_database": "7m", + "delete_database": "6m" + }, + "overrides": { + "services.90.create": "7m" + } +} diff --git a/devops/profiles/prod/profile.env b/devops/profiles/prod/profile.env new file mode 100644 index 0000000..66d5053 --- /dev/null +++ b/devops/profiles/prod/profile.env @@ -0,0 +1,20 @@ +# Stand profile: PROD +NUBES_API_ENDPOINT="https://deck-api.ngcloud.ru/api/v1" +TOKEN_FILE="secrets/prod.token" + +# Release versions +RELEASE_VERSION="2.1.23" +PROVIDER_VERSION="2.1.23" +DOCS_VERSION="2.1.23" + +# Registry/S3 settings +REGISTRY_HOST="terra.k8c.ru" +REGISTRY_HOSTNAME="terra.k8c.ru" +NAMESPACE="nubes" +PROVIDER_NAME="nubes" +S3_BUCKET="terraform-registry" +S3CFG_REGISTRY="secrets/.s3cfg_registry" + +# Signing keys +GPG_PRIVATE_KEY_PATH="secrets/private_key.asc" +GPG_PUBLIC_KEY_PATH="secrets/public_key.asc" diff --git a/devops/profiles/prod/services_list.txt b/devops/profiles/prod/services_list.txt new file mode 100644 index 0000000..5706525 --- /dev/null +++ b/devops/profiles/prod/services_list.txt @@ -0,0 +1,46 @@ +# service_id service_name # RU description +# do not delete lines; to exclude a service, comment it with # +1 dummy # Болванка для тестов +2 template # Болванка для тестирования +12 s3 # Масштабируемое хранилище данных +13 s3bucket # Бакет в сервисе S3 +19 vcOrg # Организация в Cloud Director +20 vcOrgSaas # Без интеграции Keycloak +21 vc_vdc # Выделение вычислительных ресурсов +22 vc_nsxt # Обеспечение сетевой доступности в тенанте +23 vc_vm # Виртуальная машина без каких-либо особенностей.
Данная услуга запускает vc_ip для выдачи белого адреса и подключения к ВМ +# 24 DEPRECATED Правила маршрутизации для VM (vc_nat) # excluded from active generation +25 vcexternalip # Резервирование IP адреса +26 vapp # Группировка виртуальных машин +27 vc_vm_v2 # Виртуальная машина с возможностью настройки Cloud Init
Можно развернуть в услуге несколько ВМ
В зависимости от настроек можно выдавать на ВМ vip + dns +28 vc_vm_v3 # Создание виртуальной машины +29 vcVdcGroup # Обеспечение сетевой доступности между датацентрами +32 vmpostgre # Сервис настройки PostgreSQL для виртуальной машины.
Требует наличия ВМ +50 nextcloud # Платформа совместной работы +81 superset # Сервер отчетов +82 harbor # Реестр для хранения образов контейнеров +88 k8sZitiController # Контроллер управления Client to service & service to service Zero trust network
+89 flask # Flask +90 postgres # Управляемая база данных PostgreSQL +91 redis # Управляемая база данных Redis +92 mongodb # Управляемая база данных MongoDB +93 rabbitmq # Сервер очередей +94 lucee # Cервер приложений +95 nodejs # Cервер приложений +96 pgadmin # Административный интерфейс для PostgreSQL +97 nodered # Low-code платформа для IoT +98 http # Простой HTTP контейнер +99 gitea # Сервис контроля версий +100 openwhisk # Добавление и удаления пользователя (учетной записи, тенанта) для работы с Openwhisk +110 dnszone # Хостинг DNS зоны +111 dnsrecord # DNS запись в созданной зоне +112 tenant # Организация доступов к мониторингу +113 vcComplex # Создание готовой иерархии ресурсов +114 GiteaComplex # Комплексная услуга из ["gittea", "postgres"] +115 mariadb # Управляемая база данных MySQL +116 kafka # Сервис создания кластера Apache Kafka +117 nifi # ETL-система +119 Akhq # Web интерфейс Akhq для работы с Apache Kafka +120 clickhouse # Управляемая база данных ClickHouse +149 valoTenant # Корпоративная система обмена и синхронизации файлов +150 k8sSthutrvalCluster # Платформа управления K8s
diff --git a/devops/profiles/test/operation_timeouts.json b/devops/profiles/test/operation_timeouts.json new file mode 100644 index 0000000..cbf0baa --- /dev/null +++ b/devops/profiles/test/operation_timeouts.json @@ -0,0 +1,17 @@ +{ + "default_timeout": "10m", + "idle_timeout": "10m", + "operations": { + "create": "10m", + "modify": "7m", + "resume": "7m", + "suspend": "7m", + "create_user": "7m", + "delete_user": "6m", + "create_database": "7m", + "delete_database": "6m" + }, + "overrides": { + "services.90.create": "7m" + } +} diff --git a/devops/profiles/test/profile.env b/devops/profiles/test/profile.env new file mode 100644 index 0000000..dd1bbcf --- /dev/null +++ b/devops/profiles/test/profile.env @@ -0,0 +1,20 @@ +# Stand profile: TEST +NUBES_API_ENDPOINT="https://deck-api-test.ngcloud.ru/api/v1" +TOKEN_FILE="secrets/test.token" + +# Release versions +RELEASE_VERSION="5.0.55" +PROVIDER_VERSION="5.0.55" +DOCS_VERSION="5.0.55" + +# Registry/S3 settings +REGISTRY_HOST="terra.k8c.ru" +REGISTRY_HOSTNAME="terra.k8c.ru" +NAMESPACE="nubes" +PROVIDER_NAME="nubes" +S3_BUCKET="terraform-registry" +S3CFG_REGISTRY="secrets/.s3cfg_registry" + +# Signing keys +GPG_PRIVATE_KEY_PATH="secrets/private_key.asc" +GPG_PUBLIC_KEY_PATH="secrets/public_key.asc" diff --git a/devops/profiles/test/services_list.txt b/devops/profiles/test/services_list.txt new file mode 100644 index 0000000..5706525 --- /dev/null +++ b/devops/profiles/test/services_list.txt @@ -0,0 +1,46 @@ +# service_id service_name # RU description +# do not delete lines; to exclude a service, comment it with # +1 dummy # Болванка для тестов +2 template # Болванка для тестирования +12 s3 # Масштабируемое хранилище данных +13 s3bucket # Бакет в сервисе S3 +19 vcOrg # Организация в Cloud Director +20 vcOrgSaas # Без интеграции Keycloak +21 vc_vdc # Выделение вычислительных ресурсов +22 vc_nsxt # Обеспечение сетевой доступности в тенанте +23 vc_vm # Виртуальная машина без каких-либо особенностей.
Данная услуга запускает vc_ip для выдачи белого адреса и подключения к ВМ +# 24 DEPRECATED Правила маршрутизации для VM (vc_nat) # excluded from active generation +25 vcexternalip # Резервирование IP адреса +26 vapp # Группировка виртуальных машин +27 vc_vm_v2 # Виртуальная машина с возможностью настройки Cloud Init
Можно развернуть в услуге несколько ВМ
В зависимости от настроек можно выдавать на ВМ vip + dns +28 vc_vm_v3 # Создание виртуальной машины +29 vcVdcGroup # Обеспечение сетевой доступности между датацентрами +32 vmpostgre # Сервис настройки PostgreSQL для виртуальной машины.
Требует наличия ВМ +50 nextcloud # Платформа совместной работы +81 superset # Сервер отчетов +82 harbor # Реестр для хранения образов контейнеров +88 k8sZitiController # Контроллер управления Client to service & service to service Zero trust network
+89 flask # Flask +90 postgres # Управляемая база данных PostgreSQL +91 redis # Управляемая база данных Redis +92 mongodb # Управляемая база данных MongoDB +93 rabbitmq # Сервер очередей +94 lucee # Cервер приложений +95 nodejs # Cервер приложений +96 pgadmin # Административный интерфейс для PostgreSQL +97 nodered # Low-code платформа для IoT +98 http # Простой HTTP контейнер +99 gitea # Сервис контроля версий +100 openwhisk # Добавление и удаления пользователя (учетной записи, тенанта) для работы с Openwhisk +110 dnszone # Хостинг DNS зоны +111 dnsrecord # DNS запись в созданной зоне +112 tenant # Организация доступов к мониторингу +113 vcComplex # Создание готовой иерархии ресурсов +114 GiteaComplex # Комплексная услуга из ["gittea", "postgres"] +115 mariadb # Управляемая база данных MySQL +116 kafka # Сервис создания кластера Apache Kafka +117 nifi # ETL-система +119 Akhq # Web интерфейс Akhq для работы с Apache Kafka +120 clickhouse # Управляемая база данных ClickHouse +149 valoTenant # Корпоративная система обмена и синхронизации файлов +150 k8sSthutrvalCluster # Платформа управления K8s