examples/VM: sless_job provisioning (packages, nginx, docker)

- Add sless.tf: three sless_job resources with depends_on chain - vm-install-packages: jq, python3-pip, htop, unzip - vm-install-nginx: nginx 1.18 (HTTP 200 verified) - vm-install-docker: Docker CE 29.3.1 + Compose v5.1.1 - Add functions/install-{packages,nginx,docker}/handler.py - _wait_apt_lock: cloud-init status --wait + systemctl mask fixes Ubuntu 22.04 first-boot apt lock (unattended-upgrades) - DPkg::Lock::Timeout=600 on all apt-get calls - Add variables.tf (install_run_id, vm_public_key, vm_private_key) - Add outputs.tf (install_*_result) - Bump nubes provider 5.0.49 -> 5.0.51 - doc/progress.md: document step 6 with bug analysis
2026-03-29 19:54:00 +03:00
parent b4c2e7f6b1
commit fef441681e
13 changed files with 1160 additions and 8 deletions
@@ -0,0 +1,133 @@
+# 2026-03-29 — handler.py: установка apt-пакетов на ВМ по SSH.
+# sless_job runtime: python3.11, entrypoint: handler.install
+#
+# event_json:
+#   packages: ["git", "curl", ...]  — список пакетов (обязательно)
+#   update:   true/false            — apt-get update перед install (default: true)
+#
+# env_vars:
+#   VM_IP:    внешний IP ВМ
+#   SSH_USER: логин (ubuntu)
+#   SSH_KEY:  содержимое приватного SSH-ключа (PEM)
+
+import os, io, time
+import paramiko
+
+
+def _load_key(content):
+    """Загрузить SSH-ключ (Ed25519 / RSA / ECDSA)."""
+    for cls in (paramiko.Ed25519Key, paramiko.RSAKey, paramiko.ECDSAKey):
+        try:
+            return cls.from_private_key(io.StringIO(content))
+        except Exception:
+            pass
+    raise ValueError("Неподдерживаемый тип SSH-ключа")
+
+
+def _ssh_connect(retries=5, delay=10):
+    """Подключение к ВМ с retry — ВМ может ещё загружаться."""
+    key = _load_key(os.environ["SSH_KEY"])
+    client = paramiko.SSHClient()
+    client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    last_err = None
+    for attempt in range(retries):
+        try:
+            client.connect(
+                hostname=os.environ["VM_IP"],
+                username=os.environ["SSH_USER"],
+                pkey=key,
+                timeout=15,
+            )
+            return client
+        except Exception as e:
+            last_err = e
+            if attempt < retries - 1:
+                time.sleep(delay)
+    raise RuntimeError(f"SSH не удалось после {retries} попыток: {last_err}")
+
+
+def _run(client, cmd, timeout=120, check=True):
+    """Выполнить команду, вернуть (exit_code, stdout, stderr)."""
+    _, stdout, stderr = client.exec_command(cmd, timeout=timeout)
+    code = stdout.channel.recv_exit_status()
+    out = stdout.read().decode(errors="replace").strip()
+    err = stderr.read().decode(errors="replace").strip()
+    if check and code != 0:
+        raise RuntimeError(f"Ошибка (exit {code}):\n{cmd}\nstderr: {err}")
+    return code, out, err
+
+
+def _wait_apt_lock(client, attempts=20, delay=10):
+    """Ждать завершения cloud-init и убить авто-обновления. Ubuntu 22.04+."""
+    # Шаг 1: Ждём завершения cloud-init — он держит apt при первом старте VM
+    _run(client, "timeout 300 sudo cloud-init status --wait 2>/dev/null; true", check=False, timeout=310)
+    # Шаг 2: Mask (не просто disable) — systemd не сможет перезапустить
+    _run(client, "sudo systemctl mask unattended-upgrades apt-daily.service apt-daily-upgrade.service apt-daily.timer apt-daily-upgrade.timer 2>/dev/null; true", check=False)
+    _run(client, "sudo systemctl stop unattended-upgrades apt-daily.service apt-daily-upgrade.service 2>/dev/null; true", check=False)
+    # Шаг 3: Добить оставшиеся apt/dpkg процессы
+    _run(client, "sudo pkill -9 -x unattended-upgrades apt-get apt dpkg 2>/dev/null; true", check=False)
+    _run(client, "sudo kill -9 $(sudo lsof -t /var/lib/dpkg/lock-frontend 2>/dev/null) 2>/dev/null; true", check=False)
+    # Шаг 4: Убрать стейл-локи и починить dpkg
+    _run(client, "sudo rm -f /var/lib/dpkg/lock-frontend /var/lib/dpkg/lock /var/cache/apt/archives/lock /var/lib/apt/lists/lock 2>/dev/null; true", check=False)
+    _run(client, "sudo dpkg --configure -a 2>/dev/null; true", check=False)
+    time.sleep(3)
+
+    locks = [
+        "/var/lib/dpkg/lock-frontend",
+        "/var/lib/dpkg/lock",
+        "/var/lib/apt/lists/lock",
+    ]
+    for i in range(attempts):
+        all_free = all(
+            _run(client, f"sudo flock -n {lock} true 2>/dev/null", check=False)[0] == 0
+            for lock in locks
+        )
+        if all_free:
+            return
+        # Повторить убийство процессов удерживающих lock
+        _run(client, "sudo pkill -9 -x apt-get apt dpkg 2>/dev/null; true", check=False)
+        _run(client, "sudo kill -9 $(sudo lsof -t /var/lib/dpkg/lock-frontend 2>/dev/null) 2>/dev/null; true", check=False)
+        if i < attempts - 1:
+            time.sleep(delay)
+    raise RuntimeError("apt lock занят слишком долго — проверьте процессы на ВМ")
+
+
+def install(event):
+    """Установить apt-пакеты. Идемпотентно — повторный запуск безопасен."""
+    packages = event.get("packages", [])
+    if not packages:
+        return {"status": "skipped", "reason": "packages list is empty"}
+
+    do_update = event.get("update", True)
+
+    client = _ssh_connect()
+    try:
+        _wait_apt_lock(client)
+
+        if do_update:
+            _run(client, "sudo DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=600 update -qq", timeout=420)
+
+        pkg_str = " ".join(packages)
+        _run(
+            client,
+            f"sudo DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=600 install -y -qq {pkg_str}",
+            timeout=300,
+        )
+
+        # Проверить что установилось
+        installed, missing = [], []
+        for pkg in packages:
+            code, _, _ = _run(
+                client,
+                f"dpkg -l {pkg} 2>/dev/null | grep -q '^ii'",
+                check=False,
+            )
+            (installed if code == 0 else missing).append(pkg)
+
+        return {
+            "status": "ok" if not missing else "partial",
+            "installed": installed,
+            "missing": missing,
+        }
+    finally:
+        client.close()