From 4ac21d9367d6b3f6223c78dd4c1c23e0e2e7d1dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CNaeel=E2=80=9D?= Date: Tue, 7 Jul 2026 08:27:42 +0400 Subject: [PATCH] v5.0.63: instance-level mutex to prevent parallel operations on same resource --- .../2026-07-06_architectural_refactoring.md | 48 ++++++++++ TEST_STAND/POSTGRES/main.tf | 2 +- .../{resources.tf => nubes_postgres.tf} | 6 -- TEST_STAND/POSTGRES/userUNDdb.tf | 96 +++++++++++++++++++ TOOLS/config/test/profile.env | 2 +- provider/internal/core/client.go | 14 +++ provider/internal/resources_core/crud.go | 17 ++++ .../service_operation_resource.go | 6 ++ provider/main.go | 2 +- 9 files changed, 184 insertions(+), 9 deletions(-) rename TEST_STAND/POSTGRES/{resources.tf => nubes_postgres.tf} (83%) create mode 100644 TEST_STAND/POSTGRES/userUNDdb.tf diff --git a/HISTORY/2026-07-06_architectural_refactoring.md b/HISTORY/2026-07-06_architectural_refactoring.md index 8001e35..0e8c17e 100644 --- a/HISTORY/2026-07-06_architectural_refactoring.md +++ b/HISTORY/2026-07-06_architectural_refactoring.md @@ -128,3 +128,51 @@ tf_provider/ **Остаётся риск:** - `RunInstanceOperationUniversalByCode` (client.go:1399) всё ещё делает GET — может упасть для service_operation_resource на проблемных инстансах - `CreateGenericInstanceUniversalV6` (client.go:224) — CREATE на инстансах где resourceRealm = Struct может упасть + +--- + +## ⚠️ BUG: Параллельные операции на одном инстансе ломаются + +**Симптом:** При создании нескольких subresource'ов (user, database) на одном postgres-инстансе: +- `ошибка API 500: Cannot connect to the orchestrator` — бэкенд не справляется с параллельными запросами +- `операция завершилась успешно, но объект не найден в state_out` — user создался, но RefreshResourceState не видит его +- `экземпляр не готов: операция в ожидании` — инстанс занят предыдущей операцией + +**Причина:** Terraform по умолчанию параллелит до 10 ресурсов (`-parallelism=10`). Все операции на одном инстансе (5 user'ов + 7 database'ов) стартуют одновременно → гонка на бэкенде. + +**Пример:** 2026-07-07 на инстансе `5111108D`: +- 5 user'ов создавались параллельно: user_2 ok, user_1+user_3 — orchestrator error, user_4+user_5 — state_out не найден +- 7 database'ов упали: инстанс не готов + +**План исправления:** mutex map в провайдере (стандартный подход как в AWS/GCP провайдерах): + +```go +// client.go — глобальная карта мьютексов +var instanceMutexes sync.Map // key: instanceUid + +func (c *UniversalClient) lockInstance(instanceUid string) func() { + mu, _ := c.instanceMutexes.LoadOrStore(instanceUid, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + return func() { mu.(*sync.Mutex).Unlock() } +} +``` + +Вызывать в каждом CRUD: +```go +unlock := client.lockInstance(instanceID) +defer unlock() +``` + +**Альтернативы (хуже):** +- `depends_on` цепочкой в конфиге — неудобно, требует ручной правки .tf +- `-parallelism=1` — замедляет ВСЕ ресурсы, не только subresource'ы + +**Затрагивает:** +- Все subresource-операции (postgres_user, postgres_database, mariadb_user, clickhouse_user, etc.) +- Modify + delete на одном инстансе тоже могут столкнуться + +**Статус:** ⚠️ OPEN — не исправлено +**Файлы для правки:** +- `/home/naeel/tf_provider/provider/internal/core/client.go` — добавить mutex map + lockInstance +- `/home/naeel/tf_provider/provider/internal/resources_core/crud.go` — добавить lock/unlock в Create/Update/Delete +- `/home/naeel/tf_provider/provider/internal/resources_core/service_operation_resource.go` — добавить lock/unlock diff --git a/TEST_STAND/POSTGRES/main.tf b/TEST_STAND/POSTGRES/main.tf index db58511..388a522 100644 --- a/TEST_STAND/POSTGRES/main.tf +++ b/TEST_STAND/POSTGRES/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { nubes = { source = "terra.k8c.ru/nubes-test/nubes" - version = "5.0.62" + version = "5.0.63" } } } diff --git a/TEST_STAND/POSTGRES/resources.tf b/TEST_STAND/POSTGRES/nubes_postgres.tf similarity index 83% rename from TEST_STAND/POSTGRES/resources.tf rename to TEST_STAND/POSTGRES/nubes_postgres.tf index a2b4461..1c85902 100644 --- a/TEST_STAND/POSTGRES/resources.tf +++ b/TEST_STAND/POSTGRES/nubes_postgres.tf @@ -50,11 +50,5 @@ resource "nubes_postgres" "npg" { adopt_existing_on_create = true } -resource "nubes_postgres_user" "pg_user" { - postgres_id = nubes_postgres.npg.id - username = "user0" - role = "ddl_user" - adopt_existing_on_create = true -} diff --git a/TEST_STAND/POSTGRES/userUNDdb.tf b/TEST_STAND/POSTGRES/userUNDdb.tf new file mode 100644 index 0000000..c6bc8a8 --- /dev/null +++ b/TEST_STAND/POSTGRES/userUNDdb.tf @@ -0,0 +1,96 @@ +# ============================================================================= +# PostgreSQL — пользователи (6 шт.) +# ============================================================================= +resource "nubes_postgres_user" "pg_user" { + postgres_id = nubes_postgres.npg.id + username = "user0" + role = "ddl_user" + adopt_existing_on_create = true +} + +resource "nubes_postgres_user" "pg_user_1" { + postgres_id = nubes_postgres.npg.id + username = "user1" + role = "ddl_user" + adopt_existing_on_create = true +} + +resource "nubes_postgres_user" "pg_user_2" { + postgres_id = nubes_postgres.npg.id + username = "user2" + role = "ddl_user" + adopt_existing_on_create = true +} + +resource "nubes_postgres_user" "pg_user_3" { + postgres_id = nubes_postgres.npg.id + username = "user3" + role = "ddl_user" + adopt_existing_on_create = true +} + +resource "nubes_postgres_user" "pg_user_4" { + postgres_id = nubes_postgres.npg.id + username = "user4" + role = "ddl_user" + adopt_existing_on_create = true +} + +resource "nubes_postgres_user" "pg_user_5" { + postgres_id = nubes_postgres.npg.id + username = "user5" + role = "ddl_user" + adopt_existing_on_create = true +} + +# ============================================================================= +# PostgreSQL — базы данных (7 шт.) +# ============================================================================= +resource "nubes_postgres_database" "pg_db_1" { + postgres_id = nubes_postgres.npg.id + db_name = "dbapp1" + db_owner = nubes_postgres_user.pg_user_1.username + adopt_existing_on_create = true +} + +resource "nubes_postgres_database" "pg_db_2" { + postgres_id = nubes_postgres.npg.id + db_name = "dbapp2" + db_owner = nubes_postgres_user.pg_user_2.username + adopt_existing_on_create = true +} + +resource "nubes_postgres_database" "pg_db_3" { + postgres_id = nubes_postgres.npg.id + db_name = "dbapp3" + db_owner = nubes_postgres_user.pg_user_3.username + adopt_existing_on_create = true +} + +resource "nubes_postgres_database" "pg_db_4" { + postgres_id = nubes_postgres.npg.id + db_name = "dbapp4" + db_owner = nubes_postgres_user.pg_user_4.username + adopt_existing_on_create = true +} + +resource "nubes_postgres_database" "pg_db_5" { + postgres_id = nubes_postgres.npg.id + db_name = "dbapp5" + db_owner = nubes_postgres_user.pg_user_5.username + adopt_existing_on_create = true +} + +resource "nubes_postgres_database" "pg_db_6" { + postgres_id = nubes_postgres.npg.id + db_name = "dbapp6" + db_owner = nubes_postgres_user.pg_user_1.username + adopt_existing_on_create = true +} + +resource "nubes_postgres_database" "pg_db_7" { + postgres_id = nubes_postgres.npg.id + db_name = "dbapp7" + db_owner = nubes_postgres_user.pg_user_2.username + adopt_existing_on_create = true +} diff --git a/TOOLS/config/test/profile.env b/TOOLS/config/test/profile.env index da033c4..a941473 100644 --- a/TOOLS/config/test/profile.env +++ b/TOOLS/config/test/profile.env @@ -3,7 +3,7 @@ NUBES_API_ENDPOINT="https://lk-api-gateway-test.ngcloud.ru/api/v1/svc" TOKEN_FILE="secrets/test.token" # Version -VERSION="5.0.62" +VERSION="5.0.63" # Docs generation — ONLY from docs_gen// (never from docs/) DOCS_GEN_DIR="provider/docs_gen/test" diff --git a/provider/internal/core/client.go b/provider/internal/core/client.go index a53b0e1..6dec4de 100644 --- a/provider/internal/core/client.go +++ b/provider/internal/core/client.go @@ -21,6 +21,7 @@ import ( "os" "regexp" "strings" + "sync" "time" ) @@ -36,6 +37,19 @@ type UniversalClient struct { LogLevel string } +// instanceMutexes — глобальная карта мьютексов для сериализации операций на одном инстансе. +// Terraform параллелит ресурсы (до 10), но API Nubes не поддерживает параллельные операции +// на одном инстансе (orchestrator error, state_out inconsistency). +var instanceMutexes sync.Map + +// LockInstance блокирует мьютекс для указанного instanceUid. +// Возвращает функцию unlock, которую нужно вызывать через defer. +func (c *UniversalClient) LockInstance(instanceUid string) func() { + mu, _ := instanceMutexes.LoadOrStore(instanceUid, &sync.Mutex{}) + mu.(*sync.Mutex).Lock() + return func() { mu.(*sync.Mutex).Unlock() } +} + // isProxyAPI returns true if ApiEndpoint uses legacy ?endpoint= proxy pattern (contains "index.cfm"). func (c *UniversalClient) isProxyAPI() bool { return strings.Contains(c.ApiEndpoint, "index.cfm") diff --git a/provider/internal/resources_core/crud.go b/provider/internal/resources_core/crud.go index 56a486e..a15265f 100644 --- a/provider/internal/resources_core/crud.go +++ b/provider/internal/resources_core/crud.go @@ -51,6 +51,9 @@ func UpdateResourceWithTimeout(ctx context.Context, client *core.UniversalClient if strings.TrimSpace(instanceID) == "" { return fmt.Errorf("missing instance id for modify") } + unlock := client.LockInstance(instanceID) + defer unlock() + ctxWithTimeout, err := core.WithOperationTimeout(ctx, operationTimeout) if err != nil { return err @@ -70,6 +73,11 @@ func DeleteResourceWithTimeout(ctx context.Context, client *core.UniversalClient mode = "state_only" } + if mode != "state_only" && mode != "detach" { + unlock := client.LockInstance(instanceID) + defer unlock() + } + ctxWithTimeout, err := core.WithOperationTimeout(ctx, operationTimeout) if err != nil { return err @@ -121,6 +129,9 @@ func RunOperationByCodeWithTimeout(ctx context.Context, client *core.UniversalCl if client == nil { return fmt.Errorf("missing client for operation") } + unlock := client.LockInstance(instanceID) + defer unlock() + ctxWithTimeout, err := core.WithOperationTimeout(ctx, operationTimeout) if err != nil { return err @@ -143,10 +154,13 @@ func adoptExistingInstanceOnCreate(ctx context.Context, client *core.UniversalCl switch state { case StateNotCreated: // P0.2: авто-cleanup orphan-инстанса при adopt_existing_on_create=true. + unlock := client.LockInstance(existing.InstanceUid) if err := client.RunInstanceOperationUniversal(ctx, existing.InstanceUid, "delete", nil); err != nil { + unlock() return "", fmt.Errorf("не удалось авто-удалить orphan-инстанс %s (статус: %s): %w. %s", existing.InstanceUid, statusText, err, formatInstanceDetails(existing, serviceID, displayName)) } + unlock() return client.CreateGenericInstanceUniversalV6(ctx, serviceID, displayName, params) case StateRunning, StateRunningPending: @@ -167,9 +181,12 @@ func adoptExistingInstanceOnCreate(ctx context.Context, client *core.UniversalCl if len(mismatches) > 0 { return "", fmt.Errorf("required params mismatch for resource_name %s: %s. %s", displayName, FormatRequiredParamMismatchMessage(mismatches), formatInstanceDetails(existing, serviceID, displayName)) } + unlock := client.LockInstance(existing.InstanceUid) if err := client.RunInstanceOperationUniversal(ctx, existing.InstanceUid, "resume", nil); err != nil { + unlock() return "", err } + unlock() resumed, err := client.GetInstanceState(ctx, existing.InstanceUid) if err != nil { return "", err diff --git a/provider/internal/resources_core/service_operation_resource.go b/provider/internal/resources_core/service_operation_resource.go index 93405e0..976bff3 100644 --- a/provider/internal/resources_core/service_operation_resource.go +++ b/provider/internal/resources_core/service_operation_resource.go @@ -83,6 +83,9 @@ func (r *ServiceOperationResource) Create(ctx context.Context, req resource.Crea return } + unlock := r.client.LockInstance(instanceUID) + defer unlock() + if err := r.client.RunInstanceOperationUniversalByCode(ctx, instanceUID, operation, params); err != nil { resp.Diagnostics.AddError("Ошибка клиента", err.Error()) return @@ -127,6 +130,9 @@ func (r *ServiceOperationResource) Update(ctx context.Context, req resource.Upda return } + unlock := r.client.LockInstance(instanceUID) + defer unlock() + if err := r.client.RunInstanceOperationUniversalByCode(ctx, instanceUID, operation, params); err != nil { resp.Diagnostics.AddError("Ошибка клиента", err.Error()) return diff --git a/provider/main.go b/provider/main.go index b88e3eb..ab6916f 100644 --- a/provider/main.go +++ b/provider/main.go @@ -17,7 +17,7 @@ import ( ) var ( - version string = "5.0.62" + version string = "5.0.63" ) func main() {