refactor: restructure project — separate tools, config, generated, provider
TOOLS/ — generators + scripts + config + ARCHITECTURE.md ├── yaml-generator/ (code) ├── resource-generator/ (code) ├── docs-generator/ (code) ├── scripts/ (← devops/*.sh) ├── config/ (← devops/profiles/ + devops/config/) │ ├── test/ profile.env, services_list.txt, operation_timeouts.json │ ├── prod/ │ └── dev/ └── ARCHITECTURE.md (← devops/ARCHITECTURE.md) generated/ — pipeline output only (gitignored) ├── test/resources_yaml/, go/, docs/, provider_build/ ├── prod/ └── dev/ provider/ — code only, no generated files resources_yaml/ — DELETED (generated) internal/resources_gen/ — DELETED (generated) devops/ — removed (replaced by TOOLS/scripts + TOOLS/config + generated/) Generators now fail if NUBES_*_DIR not set (no defaults to provider/). Provider requires pipeline to populate resources_gen/ before build.
This commit is contained in:
“Naeel”
@@ -0,0 +1,331 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Универсальный сборщик state_params для сервисов тест-стенда.
|
||||
Создаёт инстанс, снимает параметры, суспендит.
|
||||
|
||||
Использование:
|
||||
python3 collect_service_params.py <service_id> [display_name]
|
||||
"""
|
||||
|
||||
import json, sys, time, urllib.request, urllib.error, urllib.parse, os, yaml, html
|
||||
|
||||
API = "https://deck-api-test.ngcloud.ru/api/v1/index.cfm"
|
||||
TOKEN = open("/home/naeel/terra/terraform/secrets/test.token").read().strip()
|
||||
OUT_DIR = "/home/naeel/terra/terraform/devops/profiles/test/generated/output_inventory"
|
||||
|
||||
HEADERS = {
|
||||
"Authorization": f"Bearer {TOKEN}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# Минимальные значения по умолчанию для стандартных параметров
|
||||
PARAM_DEFAULTS = {
|
||||
"resourceRealm": "k8s-3-sandbox-nubes-ru",
|
||||
"resourceCPU": "1000",
|
||||
"resourceMemory": "1024",
|
||||
"resourceDisk": "10",
|
||||
"resourceInstances": "1",
|
||||
"resourceInstancesShards": "1",
|
||||
"needExternalAddressMaster": "false",
|
||||
"needExternalAddressSlave": "false",
|
||||
"needCreateKeeper": "false",
|
||||
"autoScale": "false",
|
||||
"durationMs": "10000",
|
||||
"failAtStart": "false",
|
||||
"failInProgress": "false",
|
||||
"domain": "web-test-stand",
|
||||
"appVersion": "latest",
|
||||
"jsonEnv": "{}",
|
||||
"login": "admin@ngcloud.ru",
|
||||
"password": "Admin123!",
|
||||
"gitPath": "https://github.com/Foxyhhd/Baldurs-Gate-test.git",
|
||||
# Для dummy
|
||||
"mapExample": "{}",
|
||||
"jsonExample": "{}",
|
||||
"yamlExample": "",
|
||||
"bodymessage": "",
|
||||
"whereFail": "1",
|
||||
}
|
||||
|
||||
# Переопределения для конкретных сервисов {service_id: {code: value}}
|
||||
SVC_PARAM_OVERRIDES = {
|
||||
97: {"domain": "inv-nodered"},
|
||||
110: {"zoneEmail": "admin@ngcloud.ru", "zoneName": "inv-test.ngcloud.ru"},
|
||||
117: {"domain": "inv-nifi"},
|
||||
99: {"domain": "inv-gitea"},
|
||||
82: {"domain": "inv-harbor"},
|
||||
}
|
||||
|
||||
def req(method, path, body=None):
|
||||
url = API + path
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
r = urllib.request.Request(url, data=data, headers=HEADERS, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(r, timeout=30) as resp:
|
||||
raw = resp.read()
|
||||
loc = resp.headers.get("Location", "")
|
||||
return json.loads(raw) if raw else {}, loc
|
||||
except urllib.error.HTTPError as e:
|
||||
raw = e.read()
|
||||
print(f" HTTP {e.code} {method} {path}: {raw[:300]}")
|
||||
return None, ""
|
||||
|
||||
def get(path):
|
||||
result, _ = req("GET", path)
|
||||
return result
|
||||
|
||||
def post(path, body):
|
||||
return req("POST", path, body)
|
||||
|
||||
def wait_op(op_uid, timeout=300):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
data = get(f"/instanceOperations/{op_uid}?fields=dtFinish,isSuccessful,errorLog,isInProgress,isPending")
|
||||
if data is None:
|
||||
return False, "request error"
|
||||
op = data.get("instanceOperation", data)
|
||||
in_prog = op.get("isInProgress", True)
|
||||
pending = op.get("isPending", True)
|
||||
if not in_prog and not pending:
|
||||
success = op.get("isSuccessful", False)
|
||||
err = op.get("errorLog", "")
|
||||
return success, err
|
||||
time.sleep(5)
|
||||
return False, "timeout"
|
||||
|
||||
def create_instance(service_id, display_name):
|
||||
print(f"\n=== Создаём инстанс: svc={service_id}, name={display_name} ===")
|
||||
|
||||
# 0. Загружаем YAML чтобы знать code для каждого param_id
|
||||
import glob as _glob
|
||||
yaml_dir = "/home/naeel/terra/terraform/devops/profiles/test/generated/resources_yaml"
|
||||
yaml_files = _glob.glob(yaml_dir + "/*.yaml")
|
||||
param_id_to_code = {}
|
||||
for yf in yaml_files:
|
||||
ydata = yaml.safe_load(open(yf))
|
||||
if ydata.get("service_id") != service_id:
|
||||
continue
|
||||
for op in ydata.get("operations", []):
|
||||
for p in op.get("params", []):
|
||||
pid = p.get("id")
|
||||
code = p.get("code", "")
|
||||
if pid and code:
|
||||
param_id_to_code[pid] = code
|
||||
break
|
||||
print(f" YAML param id→code map: {len(param_id_to_code)} entries")
|
||||
# 1. Создаём инстанс
|
||||
resp, loc = post("/instances", {
|
||||
"serviceId": service_id,
|
||||
"displayName": display_name,
|
||||
"descr": "inventory-collector",
|
||||
})
|
||||
if resp is None:
|
||||
return None
|
||||
if isinstance(resp, dict):
|
||||
instance_uid = resp.get("instanceUid")
|
||||
else:
|
||||
instance_uid = None
|
||||
if not instance_uid and loc:
|
||||
instance_uid = loc.rstrip("/").split("/")[-1]
|
||||
if not instance_uid:
|
||||
print(f" ERR: нет instanceUid. resp={resp}, loc={loc}")
|
||||
return None
|
||||
print(f" instanceUid: {instance_uid}")
|
||||
|
||||
# 2. Создаём операцию create
|
||||
resp, loc = post("/instanceOperations", {
|
||||
"instanceUid": instance_uid,
|
||||
"operation": "create",
|
||||
})
|
||||
if resp is None:
|
||||
return None
|
||||
op = resp.get("instanceOperation", resp) if isinstance(resp, dict) else {}
|
||||
op_uid = op.get("instanceOperationUid")
|
||||
if not op_uid and loc:
|
||||
op_uid = loc.rstrip("/").split("/")[-1]
|
||||
if not op_uid:
|
||||
print(f" ERR: нет opUid. resp={resp}, loc={loc}")
|
||||
return None
|
||||
print(f" opUid: {op_uid}")
|
||||
|
||||
# 3. Получаем cfsParams с ID
|
||||
op_detail = get(f"/instanceOperations/{op_uid}?fields=cfsParams")
|
||||
if op_detail is None:
|
||||
return None
|
||||
cfs_params = (op_detail.get("instanceOperation") or op_detail).get("cfsParams", [])
|
||||
print(f" cfsParams count: {len(cfs_params)}")
|
||||
for p in cfs_params:
|
||||
code_display = p.get("svcOperationCfsParam") or p.get("code", "?")
|
||||
print(f" id={p['svcOperationCfsParamId']} code={code_display} required={p.get('isRequired')} default={p.get('defaultValue')} paramValue={p.get('paramValue')}")
|
||||
|
||||
# 4. Заполняем параметры (POST для КАЖДОГО — иначе /run ломается с GUID-ошибкой)
|
||||
filled = 0
|
||||
for p in cfs_params:
|
||||
param_id = p["svcOperationCfsParamId"]
|
||||
# Получаем code: сначала из прямого поля API, потом из YAML-маппинга
|
||||
code = p.get("svcOperationCfsParam") or param_id_to_code.get(param_id, p.get("code", ""))
|
||||
current = p.get("paramValue")
|
||||
default = p.get("defaultValue")
|
||||
# Переопределения для конкретного сервиса имеют наивысший приоритет
|
||||
overrides = SVC_PARAM_OVERRIDES.get(service_id, {})
|
||||
|
||||
# Определяем значение
|
||||
# Пропускаем pre-populated невалидные значения ('""', пустые строки)
|
||||
if current is not None and str(current) not in ('""', "''", ""):
|
||||
value = str(current)
|
||||
elif code in overrides:
|
||||
value = overrides[code]
|
||||
elif default is not None and str(default).strip() != "":
|
||||
# HTML-unescape дефолтов (API может вернуть " и т.п.)
|
||||
value = html.unescape(str(default))
|
||||
elif code in PARAM_DEFAULTS:
|
||||
value = PARAM_DEFAULTS[code]
|
||||
else:
|
||||
value = "" # Отправляем пустую строку — сервер требует запись для каждого параметра
|
||||
|
||||
resp2, loc2 = post("/instanceOperationCfsParams", {
|
||||
"instanceOperationUid": op_uid,
|
||||
"svcOperationCfsParamId": param_id,
|
||||
"paramValue": value,
|
||||
})
|
||||
if resp2 is not None and (loc2 or resp2 != {}):
|
||||
filled += 1
|
||||
elif resp2 is None:
|
||||
print(f" WARN: param id={param_id} code={code} не принят API (400/422), value={value!r}")
|
||||
print(f" Заполнено параметров: {filled}")
|
||||
|
||||
# 5. validate-cfs — GET (в HAR это GET, не POST; на test может вернуть 405 — терпим)
|
||||
req("GET", f"/instanceOperations/{op_uid}/validate-cfs")
|
||||
|
||||
# 6. Run
|
||||
resp4, _ = post(f"/instanceOperations/{op_uid}/run", {})
|
||||
if resp4 is None:
|
||||
print(" ERR: run вернул ошибку — сервис может быть сырым/нерабочим")
|
||||
return instance_uid # возвращаем uid — state может быть частичным
|
||||
print(" Операция запущена, ждём завершения...")
|
||||
|
||||
# 7. Ждём
|
||||
success, err = wait_op(op_uid, timeout=600)
|
||||
if not success:
|
||||
print(f" WARN: операция завершилась с ошибкой (сервис сырой?): {err[:200]}")
|
||||
# Не прерываемся — возвращаем uid, state может содержать частичные данные
|
||||
else:
|
||||
print(" Создание завершено успешно!")
|
||||
return instance_uid
|
||||
|
||||
def collect_state(instance_uid):
|
||||
detail = get(f"/instances/{instance_uid}?fields=instanceUid,displayName,serviceId,svc,state,explainedStatus")
|
||||
if not detail:
|
||||
return None
|
||||
inst = detail.get("instance", {})
|
||||
state = inst.get("state", {}) or {}
|
||||
return {
|
||||
"uid": instance_uid,
|
||||
"name": inst.get("displayName",""),
|
||||
"serviceId": inst.get("serviceId"),
|
||||
"svc_code": inst.get("svc",""),
|
||||
"status": inst.get("explainedStatus",""),
|
||||
"params": state.get("params",{}) or {},
|
||||
"out": state.get("out",{}) or {},
|
||||
"vault_keys": list((state.get("vault",{}) or {}).keys()),
|
||||
}
|
||||
|
||||
def suspend_instance(instance_uid):
|
||||
print(f" Суспендим {instance_uid}...")
|
||||
resp, loc = post("/instanceOperations", {
|
||||
"instanceUid": instance_uid,
|
||||
"operation": "suspend",
|
||||
})
|
||||
if resp is None:
|
||||
return
|
||||
op = resp.get("instanceOperation", resp) if isinstance(resp, dict) else {}
|
||||
op_uid = op.get("instanceOperationUid")
|
||||
if not op_uid and loc:
|
||||
op_uid = loc.rstrip("/").split("/")[-1]
|
||||
if not op_uid:
|
||||
return
|
||||
post(f"/instanceOperations/{op_uid}/run", {})
|
||||
success, err = wait_op(op_uid, timeout=300)
|
||||
if success:
|
||||
print(" Suspended!")
|
||||
else:
|
||||
print(f" WARN suspend: {err}")
|
||||
|
||||
def save_to_inventory(state_data):
|
||||
svc_id = state_data["serviceId"]
|
||||
safe = state_data["svc_code"].replace("/","_").replace(" ","_")
|
||||
fname = os.path.join(OUT_DIR, f"svc_{svc_id}_{safe}.json")
|
||||
|
||||
existing = {}
|
||||
if os.path.exists(fname):
|
||||
existing = json.load(open(fname))
|
||||
|
||||
instances = existing.get("instances", [])
|
||||
# Не дублируем
|
||||
uids = {i["uid"] for i in instances}
|
||||
if state_data["uid"] not in uids:
|
||||
instances.append(state_data)
|
||||
|
||||
result = {
|
||||
"serviceId": svc_id,
|
||||
"svc_code": state_data["svc_code"],
|
||||
"instances": instances,
|
||||
}
|
||||
with open(fname, "w") as f:
|
||||
json.dump(result, f, indent=2, ensure_ascii=False)
|
||||
print(f" Saved: {fname}")
|
||||
|
||||
# Обновляем индекс
|
||||
rebuild_index()
|
||||
|
||||
def rebuild_index():
|
||||
import glob
|
||||
index = []
|
||||
for f in sorted(glob.glob(os.path.join(OUT_DIR, "svc_*.json"))):
|
||||
data = json.load(open(f))
|
||||
all_params = set()
|
||||
all_out = set()
|
||||
all_vault = set()
|
||||
for item in data.get("instances", []):
|
||||
all_params.update((item.get("params") or {}).keys())
|
||||
out = item.get("out")
|
||||
all_out.update((out.keys() if isinstance(out, dict) else []))
|
||||
all_vault.update(item.get("vault_keys", []))
|
||||
index.append({
|
||||
"serviceId": data["serviceId"],
|
||||
"svc_code": data["svc_code"],
|
||||
"instance_count": len(data.get("instances", [])),
|
||||
"state_params_keys": sorted(all_params),
|
||||
"state_out_keys": sorted(all_out),
|
||||
"vault_keys": sorted(all_vault),
|
||||
})
|
||||
with open(os.path.join(OUT_DIR, "index.json"), "w") as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# ===== MAIN =====
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: collect_service_params.py <service_id> [display_name]")
|
||||
sys.exit(1)
|
||||
|
||||
service_id = int(sys.argv[1])
|
||||
display_name = sys.argv[2] if len(sys.argv) > 2 else f"inventory-svc{service_id}"
|
||||
|
||||
instance_uid = create_instance(service_id, display_name)
|
||||
if not instance_uid:
|
||||
print("FAILED: не удалось создать инстанс")
|
||||
sys.exit(1)
|
||||
|
||||
# Небольшая пауза перед чтением state
|
||||
time.sleep(3)
|
||||
state = collect_state(instance_uid)
|
||||
if state:
|
||||
print(f"\n state_params keys: {list(state['params'].keys())}")
|
||||
print(f" state_out keys: {list(state['out'].keys()) if isinstance(state['out'], dict) else '-'}")
|
||||
print(f" vault keys: {state['vault_keys']}")
|
||||
save_to_inventory(state)
|
||||
else:
|
||||
print("WARN: не удалось получить state")
|
||||
|
||||
suspend_instance(instance_uid)
|
||||
print("\nDone.")
|
||||
Reference in New Issue
Block a user