“Naeel”
dd56bcda83
TOOLS/ — generators + scripts + config + ARCHITECTURE.md ├── yaml-generator/ (code) ├── resource-generator/ (code) ├── docs-generator/ (code) ├── scripts/ (← devops/*.sh) ├── config/ (← devops/profiles/ + devops/config/) │ ├── test/ profile.env, services_list.txt, operation_timeouts.json │ ├── prod/ │ └── dev/ └── ARCHITECTURE.md (← devops/ARCHITECTURE.md) generated/ — pipeline output only (gitignored) ├── test/resources_yaml/, go/, docs/, provider_build/ ├── prod/ └── dev/ provider/ — code only, no generated files resources_yaml/ — DELETED (generated) internal/resources_gen/ — DELETED (generated) devops/ — removed (replaced by TOOLS/scripts + TOOLS/config + generated/) Generators now fail if NUBES_*_DIR not set (no defaults to provider/). Provider requires pipeline to populate resources_gen/ before build.
332 lines
13 KiB
Python
332 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Универсальный сборщик state_params для сервисов тест-стенда.
|
||
Создаёт инстанс, снимает параметры, суспендит.
|
||
|
||
Использование:
|
||
python3 collect_service_params.py <service_id> [display_name]
|
||
"""
|
||
|
||
import json, sys, time, urllib.request, urllib.error, urllib.parse, os, yaml, html
|
||
|
||
API = "https://deck-api-test.ngcloud.ru/api/v1/index.cfm"
|
||
TOKEN = open("/home/naeel/terra/terraform/secrets/test.token").read().strip()
|
||
OUT_DIR = "/home/naeel/terra/terraform/devops/profiles/test/generated/output_inventory"
|
||
|
||
HEADERS = {
|
||
"Authorization": f"Bearer {TOKEN}",
|
||
"Content-Type": "application/json",
|
||
}
|
||
|
||
# Минимальные значения по умолчанию для стандартных параметров
|
||
PARAM_DEFAULTS = {
|
||
"resourceRealm": "k8s-3-sandbox-nubes-ru",
|
||
"resourceCPU": "1000",
|
||
"resourceMemory": "1024",
|
||
"resourceDisk": "10",
|
||
"resourceInstances": "1",
|
||
"resourceInstancesShards": "1",
|
||
"needExternalAddressMaster": "false",
|
||
"needExternalAddressSlave": "false",
|
||
"needCreateKeeper": "false",
|
||
"autoScale": "false",
|
||
"durationMs": "10000",
|
||
"failAtStart": "false",
|
||
"failInProgress": "false",
|
||
"domain": "web-test-stand",
|
||
"appVersion": "latest",
|
||
"jsonEnv": "{}",
|
||
"login": "admin@ngcloud.ru",
|
||
"password": "Admin123!",
|
||
"gitPath": "https://github.com/Foxyhhd/Baldurs-Gate-test.git",
|
||
# Для dummy
|
||
"mapExample": "{}",
|
||
"jsonExample": "{}",
|
||
"yamlExample": "",
|
||
"bodymessage": "",
|
||
"whereFail": "1",
|
||
}
|
||
|
||
# Переопределения для конкретных сервисов {service_id: {code: value}}
|
||
SVC_PARAM_OVERRIDES = {
|
||
97: {"domain": "inv-nodered"},
|
||
110: {"zoneEmail": "admin@ngcloud.ru", "zoneName": "inv-test.ngcloud.ru"},
|
||
117: {"domain": "inv-nifi"},
|
||
99: {"domain": "inv-gitea"},
|
||
82: {"domain": "inv-harbor"},
|
||
}
|
||
|
||
def req(method, path, body=None):
|
||
url = API + path
|
||
data = json.dumps(body).encode() if body is not None else None
|
||
r = urllib.request.Request(url, data=data, headers=HEADERS, method=method)
|
||
try:
|
||
with urllib.request.urlopen(r, timeout=30) as resp:
|
||
raw = resp.read()
|
||
loc = resp.headers.get("Location", "")
|
||
return json.loads(raw) if raw else {}, loc
|
||
except urllib.error.HTTPError as e:
|
||
raw = e.read()
|
||
print(f" HTTP {e.code} {method} {path}: {raw[:300]}")
|
||
return None, ""
|
||
|
||
def get(path):
|
||
result, _ = req("GET", path)
|
||
return result
|
||
|
||
def post(path, body):
|
||
return req("POST", path, body)
|
||
|
||
def wait_op(op_uid, timeout=300):
|
||
deadline = time.time() + timeout
|
||
while time.time() < deadline:
|
||
data = get(f"/instanceOperations/{op_uid}?fields=dtFinish,isSuccessful,errorLog,isInProgress,isPending")
|
||
if data is None:
|
||
return False, "request error"
|
||
op = data.get("instanceOperation", data)
|
||
in_prog = op.get("isInProgress", True)
|
||
pending = op.get("isPending", True)
|
||
if not in_prog and not pending:
|
||
success = op.get("isSuccessful", False)
|
||
err = op.get("errorLog", "")
|
||
return success, err
|
||
time.sleep(5)
|
||
return False, "timeout"
|
||
|
||
def create_instance(service_id, display_name):
|
||
print(f"\n=== Создаём инстанс: svc={service_id}, name={display_name} ===")
|
||
|
||
# 0. Загружаем YAML чтобы знать code для каждого param_id
|
||
import glob as _glob
|
||
yaml_dir = "/home/naeel/terra/terraform/devops/profiles/test/generated/resources_yaml"
|
||
yaml_files = _glob.glob(yaml_dir + "/*.yaml")
|
||
param_id_to_code = {}
|
||
for yf in yaml_files:
|
||
ydata = yaml.safe_load(open(yf))
|
||
if ydata.get("service_id") != service_id:
|
||
continue
|
||
for op in ydata.get("operations", []):
|
||
for p in op.get("params", []):
|
||
pid = p.get("id")
|
||
code = p.get("code", "")
|
||
if pid and code:
|
||
param_id_to_code[pid] = code
|
||
break
|
||
print(f" YAML param id→code map: {len(param_id_to_code)} entries")
|
||
# 1. Создаём инстанс
|
||
resp, loc = post("/instances", {
|
||
"serviceId": service_id,
|
||
"displayName": display_name,
|
||
"descr": "inventory-collector",
|
||
})
|
||
if resp is None:
|
||
return None
|
||
if isinstance(resp, dict):
|
||
instance_uid = resp.get("instanceUid")
|
||
else:
|
||
instance_uid = None
|
||
if not instance_uid and loc:
|
||
instance_uid = loc.rstrip("/").split("/")[-1]
|
||
if not instance_uid:
|
||
print(f" ERR: нет instanceUid. resp={resp}, loc={loc}")
|
||
return None
|
||
print(f" instanceUid: {instance_uid}")
|
||
|
||
# 2. Создаём операцию create
|
||
resp, loc = post("/instanceOperations", {
|
||
"instanceUid": instance_uid,
|
||
"operation": "create",
|
||
})
|
||
if resp is None:
|
||
return None
|
||
op = resp.get("instanceOperation", resp) if isinstance(resp, dict) else {}
|
||
op_uid = op.get("instanceOperationUid")
|
||
if not op_uid and loc:
|
||
op_uid = loc.rstrip("/").split("/")[-1]
|
||
if not op_uid:
|
||
print(f" ERR: нет opUid. resp={resp}, loc={loc}")
|
||
return None
|
||
print(f" opUid: {op_uid}")
|
||
|
||
# 3. Получаем cfsParams с ID
|
||
op_detail = get(f"/instanceOperations/{op_uid}?fields=cfsParams")
|
||
if op_detail is None:
|
||
return None
|
||
cfs_params = (op_detail.get("instanceOperation") or op_detail).get("cfsParams", [])
|
||
print(f" cfsParams count: {len(cfs_params)}")
|
||
for p in cfs_params:
|
||
code_display = p.get("svcOperationCfsParam") or p.get("code", "?")
|
||
print(f" id={p['svcOperationCfsParamId']} code={code_display} required={p.get('isRequired')} default={p.get('defaultValue')} paramValue={p.get('paramValue')}")
|
||
|
||
# 4. Заполняем параметры (POST для КАЖДОГО — иначе /run ломается с GUID-ошибкой)
|
||
filled = 0
|
||
for p in cfs_params:
|
||
param_id = p["svcOperationCfsParamId"]
|
||
# Получаем code: сначала из прямого поля API, потом из YAML-маппинга
|
||
code = p.get("svcOperationCfsParam") or param_id_to_code.get(param_id, p.get("code", ""))
|
||
current = p.get("paramValue")
|
||
default = p.get("defaultValue")
|
||
# Переопределения для конкретного сервиса имеют наивысший приоритет
|
||
overrides = SVC_PARAM_OVERRIDES.get(service_id, {})
|
||
|
||
# Определяем значение
|
||
# Пропускаем pre-populated невалидные значения ('""', пустые строки)
|
||
if current is not None and str(current) not in ('""', "''", ""):
|
||
value = str(current)
|
||
elif code in overrides:
|
||
value = overrides[code]
|
||
elif default is not None and str(default).strip() != "":
|
||
# HTML-unescape дефолтов (API может вернуть " и т.п.)
|
||
value = html.unescape(str(default))
|
||
elif code in PARAM_DEFAULTS:
|
||
value = PARAM_DEFAULTS[code]
|
||
else:
|
||
value = "" # Отправляем пустую строку — сервер требует запись для каждого параметра
|
||
|
||
resp2, loc2 = post("/instanceOperationCfsParams", {
|
||
"instanceOperationUid": op_uid,
|
||
"svcOperationCfsParamId": param_id,
|
||
"paramValue": value,
|
||
})
|
||
if resp2 is not None and (loc2 or resp2 != {}):
|
||
filled += 1
|
||
elif resp2 is None:
|
||
print(f" WARN: param id={param_id} code={code} не принят API (400/422), value={value!r}")
|
||
print(f" Заполнено параметров: {filled}")
|
||
|
||
# 5. validate-cfs — GET (в HAR это GET, не POST; на test может вернуть 405 — терпим)
|
||
req("GET", f"/instanceOperations/{op_uid}/validate-cfs")
|
||
|
||
# 6. Run
|
||
resp4, _ = post(f"/instanceOperations/{op_uid}/run", {})
|
||
if resp4 is None:
|
||
print(" ERR: run вернул ошибку — сервис может быть сырым/нерабочим")
|
||
return instance_uid # возвращаем uid — state может быть частичным
|
||
print(" Операция запущена, ждём завершения...")
|
||
|
||
# 7. Ждём
|
||
success, err = wait_op(op_uid, timeout=600)
|
||
if not success:
|
||
print(f" WARN: операция завершилась с ошибкой (сервис сырой?): {err[:200]}")
|
||
# Не прерываемся — возвращаем uid, state может содержать частичные данные
|
||
else:
|
||
print(" Создание завершено успешно!")
|
||
return instance_uid
|
||
|
||
def collect_state(instance_uid):
|
||
detail = get(f"/instances/{instance_uid}?fields=instanceUid,displayName,serviceId,svc,state,explainedStatus")
|
||
if not detail:
|
||
return None
|
||
inst = detail.get("instance", {})
|
||
state = inst.get("state", {}) or {}
|
||
return {
|
||
"uid": instance_uid,
|
||
"name": inst.get("displayName",""),
|
||
"serviceId": inst.get("serviceId"),
|
||
"svc_code": inst.get("svc",""),
|
||
"status": inst.get("explainedStatus",""),
|
||
"params": state.get("params",{}) or {},
|
||
"out": state.get("out",{}) or {},
|
||
"vault_keys": list((state.get("vault",{}) or {}).keys()),
|
||
}
|
||
|
||
def suspend_instance(instance_uid):
|
||
print(f" Суспендим {instance_uid}...")
|
||
resp, loc = post("/instanceOperations", {
|
||
"instanceUid": instance_uid,
|
||
"operation": "suspend",
|
||
})
|
||
if resp is None:
|
||
return
|
||
op = resp.get("instanceOperation", resp) if isinstance(resp, dict) else {}
|
||
op_uid = op.get("instanceOperationUid")
|
||
if not op_uid and loc:
|
||
op_uid = loc.rstrip("/").split("/")[-1]
|
||
if not op_uid:
|
||
return
|
||
post(f"/instanceOperations/{op_uid}/run", {})
|
||
success, err = wait_op(op_uid, timeout=300)
|
||
if success:
|
||
print(" Suspended!")
|
||
else:
|
||
print(f" WARN suspend: {err}")
|
||
|
||
def save_to_inventory(state_data):
|
||
svc_id = state_data["serviceId"]
|
||
safe = state_data["svc_code"].replace("/","_").replace(" ","_")
|
||
fname = os.path.join(OUT_DIR, f"svc_{svc_id}_{safe}.json")
|
||
|
||
existing = {}
|
||
if os.path.exists(fname):
|
||
existing = json.load(open(fname))
|
||
|
||
instances = existing.get("instances", [])
|
||
# Не дублируем
|
||
uids = {i["uid"] for i in instances}
|
||
if state_data["uid"] not in uids:
|
||
instances.append(state_data)
|
||
|
||
result = {
|
||
"serviceId": svc_id,
|
||
"svc_code": state_data["svc_code"],
|
||
"instances": instances,
|
||
}
|
||
with open(fname, "w") as f:
|
||
json.dump(result, f, indent=2, ensure_ascii=False)
|
||
print(f" Saved: {fname}")
|
||
|
||
# Обновляем индекс
|
||
rebuild_index()
|
||
|
||
def rebuild_index():
|
||
import glob
|
||
index = []
|
||
for f in sorted(glob.glob(os.path.join(OUT_DIR, "svc_*.json"))):
|
||
data = json.load(open(f))
|
||
all_params = set()
|
||
all_out = set()
|
||
all_vault = set()
|
||
for item in data.get("instances", []):
|
||
all_params.update((item.get("params") or {}).keys())
|
||
out = item.get("out")
|
||
all_out.update((out.keys() if isinstance(out, dict) else []))
|
||
all_vault.update(item.get("vault_keys", []))
|
||
index.append({
|
||
"serviceId": data["serviceId"],
|
||
"svc_code": data["svc_code"],
|
||
"instance_count": len(data.get("instances", [])),
|
||
"state_params_keys": sorted(all_params),
|
||
"state_out_keys": sorted(all_out),
|
||
"vault_keys": sorted(all_vault),
|
||
})
|
||
with open(os.path.join(OUT_DIR, "index.json"), "w") as f:
|
||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||
|
||
# ===== MAIN =====
|
||
if __name__ == "__main__":
|
||
if len(sys.argv) < 2:
|
||
print("Usage: collect_service_params.py <service_id> [display_name]")
|
||
sys.exit(1)
|
||
|
||
service_id = int(sys.argv[1])
|
||
display_name = sys.argv[2] if len(sys.argv) > 2 else f"inventory-svc{service_id}"
|
||
|
||
instance_uid = create_instance(service_id, display_name)
|
||
if not instance_uid:
|
||
print("FAILED: не удалось создать инстанс")
|
||
sys.exit(1)
|
||
|
||
# Небольшая пауза перед чтением state
|
||
time.sleep(3)
|
||
state = collect_state(instance_uid)
|
||
if state:
|
||
print(f"\n state_params keys: {list(state['params'].keys())}")
|
||
print(f" state_out keys: {list(state['out'].keys()) if isinstance(state['out'], dict) else '-'}")
|
||
print(f" vault keys: {state['vault_keys']}")
|
||
save_to_inventory(state)
|
||
else:
|
||
print("WARN: не удалось получить state")
|
||
|
||
suspend_instance(instance_uid)
|
||
print("\nDone.")
|