v5.0.58: fix API format — ?endpoint= instead of REST path (DDoS-Guard 403)

This commit is contained in:
“Naeel”
2026-07-01 14:28:08 +04:00
parent 9c78cba1a0
commit d2b9e91624
19 changed files with 50 additions and 577 deletions
@@ -1,45 +0,0 @@
# S3 Event Function PoC (один apply, без создания `nubes_s3`)
Стенд делает всё в рамках одного `terraform apply`:
1. Создаёт `nubes_s3bucket` на существующем `s3_uid`.
2. Автоматически запускает demo-check (`terraform_data + local-exec`).
3. Demo-check пишет тестовый объект в бакет.
4. Событие бакета доставляется в простую функцию, и apply печатает `CONFIRMED`.
## Подготовка
- Нужны: `terraform`, `python3`, `mc`.
- В `terraform.tfvars` задайте `api_token`, `s3_uid`, `bucket_name`.
- Путь к `.s3cfg_registry` можно оставить по умолчанию или задать переменной `s3cfg_path`.
Можно взять `api_token` и `s3_uid` из `TEST_STAND/POSTGRES/terraform.tfvars`.
## Запуск (одна команда apply)
```bash
cd TEST_STAND/S3_EVENT_FUNCTION_POC
terraform init
terraform apply -auto-approve
```
## Подтверждение, что сработало
- Во время `apply` появится строка:
- `CONFIRMED: bucket event reached function ...`
- После `apply` доступен output:
- `event_demo_confirmation`
Проверить output:
```bash
terraform output event_demo_confirmation
```
## Отладка (если failed)
Смотрите файлы в каталоге стенда:
- `function_stdout.log`
- `watcher_stdout.log`
- `function.log`
@@ -1,13 +0,0 @@
CONFIRMED ✓
Source bucket : poc-s3evt-src
Dest bucket : poc-s3evt-dst
Object : event-trigger-1773587291.txt
Time : Sun Mar 15 03:08:32 PM UTC 2026
Что произошло:
1. terraform создал два бакета: poc-s3evt-src и poc-s3evt-dst
2. Скрипт написал 'event-trigger-1773587291.txt' в poc-s3evt-src
3. Срабатывание подтверждено через: fallback check
4. «Функция» скопировала объект в poc-s3evt-dst
5. Копия подтверждена — тест пройден
-51
View File
@@ -1,51 +0,0 @@
provider "nubes" {
api_token = var.api_token
api_endpoint = var.api_endpoint
}
# Бакет-источник — сюда пишем объект
resource "nubes_s3bucket" "source" {
resource_name = "${var.bucket_name}-src"
s3_user_uid = var.s3_uid
bucket_name = "${var.bucket_name}-src"
adopt_existing_on_create = true # бакет уже создан предыдущим apply
max_size = -1
read_all = false
list_all = false
cors_all = false
placement = "HOT"
}
# Бакет-назначение — сюда копирует «функция» по событию
resource "nubes_s3bucket" "dest" {
resource_name = "${var.bucket_name}-dst"
s3_user_uid = var.s3_uid
bucket_name = "${var.bucket_name}-dst"
max_size = -1
read_all = false
list_all = false
cors_all = false
placement = "HOT"
}
locals {
src_bucket = try(nubes_s3bucket.source.state_params["bucketName"], "${var.bucket_name}-src")
dst_bucket = try(nubes_s3bucket.dest.state_params["bucketName"], "${var.bucket_name}-dst")
}
# «Функция»: mc watch ловит событие в source → копирует объект в dest
resource "terraform_data" "event_delivery_check" {
depends_on = [nubes_s3bucket.source, nubes_s3bucket.dest]
triggers_replace = {
src = local.src_bucket
dst = local.dst_bucket
s3cfg_path = var.s3cfg_path
}
provisioner "local-exec" {
command = "bash ${path.module}/scripts/copy_on_event.sh \"${var.s3cfg_path}\" \"${local.src_bucket}\" \"${local.dst_bucket}\" \"${var.demo_timeout_sec}\""
}
}
@@ -1,11 +0,0 @@
output "source_bucket" {
value = nubes_s3bucket.source.state_params["bucketName"]
}
output "dest_bucket" {
value = nubes_s3bucket.dest.state_params["bucketName"]
}
output "event_demo_confirmation" {
value = try(trimspace(file("${path.module}/apply_confirmation.txt")), "confirmation file not found")
}
@@ -1,27 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
if [[ ! -f "$ROOT_DIR/terraform.tfvars" ]]; then
echo "ERROR: $ROOT_DIR/terraform.tfvars not found"
echo "Create it from terraform.tfvars.example or reuse values from TEST_STAND/POSTGRES/terraform.tfvars"
exit 1
fi
cd "$ROOT_DIR"
terraform init
terraform apply -auto-approve
BUCKET="$(terraform output -raw bucket_name 2>/dev/null || true)"
if [[ -z "$BUCKET" ]]; then
echo "ERROR: bucket_name output is empty"
exit 1
fi
echo "Bucket created: $BUCKET"
echo "Running event->function demo..."
export S3CFG="${S3CFG:-/home/naeel/remote_dev/terraform/secrets/.s3cfg_registry}"
export BUCKET
bash "$ROOT_DIR/scripts/run_demo.sh"
@@ -1,138 +0,0 @@
#!/usr/bin/env bash
# Симуляция event-driven функции:
# 1. mc watch следит за SOURCE бакетом (как S3 notification)
# 2. При появлении объекта — "функция" копирует его в DEST бакет
# 3. Подтверждение записывается в apply_confirmation.txt
#
# Аргументы: <s3cfg_path> <src_bucket> <dst_bucket> [timeout_sec]
set -euo pipefail
S3CFG="${1:?s3cfg_path required}"
SRC_BUCKET="${2:?src_bucket required}"
DST_BUCKET="${3:?dst_bucket required}"
TIMEOUT="${4:-35}"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
POC_DIR="$(dirname "$SCRIPT_DIR")"
CONFIRM_FILE="${POC_DIR}/apply_confirmation.txt"
EVENTS_FILE="/tmp/poc_events_$$.json"
ALIAS="poc$$"
cleanup() {
kill "$WATCHER_PID" 2>/dev/null || true
rm -f "$EVENTS_FILE"
mc alias rm "$ALIAS" >/dev/null 2>&1 || true
}
trap cleanup EXIT
echo "[poc] S3CFG: $S3CFG"
echo "[poc] Source bucket : $SRC_BUCKET"
echo "[poc] Dest bucket : $DST_BUCKET"
echo "[poc] Timeout : ${TIMEOUT}s"
# Парсим .s3cfg
HOST=$(grep -E '^\s*host_base\s*=' "$S3CFG" | head -1 | sed 's/.*=\s*//' | tr -d '[:space:]')
AK=$(grep -E '^\s*access_key\s*=' "$S3CFG" | head -1 | sed 's/.*=\s*//' | tr -d '[:space:]')
SK=$(grep -E '^\s*secret_key\s*=' "$S3CFG" | head -1 | sed 's/.*=\s*//' | tr -d '[:space:]')
USE_HTTPS=$(grep -E '^\s*use_https\s*=' "$S3CFG" | head -1 | sed 's/.*=\s*//' | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
if [[ -z "$HOST" || -z "$AK" || -z "$SK" ]]; then
echo "[poc] ERROR: не удалось разобрать .s3cfg" >&2; exit 1
fi
SCHEME="https"
if [[ "$USE_HTTPS" == "false" || "$USE_HTTPS" == "0" || "$USE_HTTPS" == "no" ]]; then
SCHEME="http"
fi
if [[ "$HOST" == http://* || "$HOST" == https://* ]]; then
ENDPOINT="$HOST"
else
ENDPOINT="${SCHEME}://${HOST}"
fi
# Настраиваем mc alias
mc alias set "$ALIAS" "$ENDPOINT" "$AK" "$SK" --api S3v4 >/dev/null 2>&1
echo "[poc] mc alias '$ALIAS' → ${ENDPOINT}"
# Запускаем mc watch на SOURCE (стримит события в файл)
touch "$EVENTS_FILE"
mc watch --json "${ALIAS}/${SRC_BUCKET}" >"$EVENTS_FILE" 2>&1 &
WATCHER_PID=$!
echo "[poc] mc watch запущен (pid=$WATCHER_PID)"
sleep 2 # даём watch время подключиться
# Имя триггерного объекта
TRIGGER_KEY="event-trigger-$(date +%s).txt"
echo "[poc] Пишем объект '${TRIGGER_KEY}' в ${SRC_BUCKET} (trigger)..."
echo "S3 event PoC trigger — $(date -u)" | mc pipe "${ALIAS}/${SRC_BUCKET}/${TRIGGER_KEY}"
echo "[poc] Объект записан. Ждём события через mc watch (макс ${TIMEOUT}s)..."
# Ждём появления ключа в потоке событий mc watch
ELAPSED=0
EVENT_DETECTED=false
EVENT_METHOD=""
while [[ $ELAPSED -lt $TIMEOUT ]]; do
if grep -q "${TRIGGER_KEY}" "$EVENTS_FILE" 2>/dev/null; then
EVENT_DETECTED=true
EVENT_METHOD="mc watch"
echo "[poc] Событие обнаружено в потоке mc watch (через ${ELAPSED}s)"
break
fi
sleep 2
ELAPSED=$((ELAPSED + 2))
done
if [[ "$EVENT_DETECTED" != "true" ]]; then
echo "[poc] WARN: mc watch не поймал объект за ${TIMEOUT}s, проверяем напрямую..."
if mc ls "${ALIAS}/${SRC_BUCKET}/${TRIGGER_KEY}" >/dev/null 2>&1; then
EVENT_DETECTED=true
EVENT_METHOD="fallback check"
echo "[poc] Объект есть в source — выполняем копирование (fallback)"
fi
fi
if [[ "$EVENT_DETECTED" != "true" ]]; then
echo "[poc] FAILED: объект не найден в source bucket" >&2
cat >"$CONFIRM_FILE" <<EOF
FAILED
Объект ${TRIGGER_KEY} не появился в ${SRC_BUCKET} за ${TIMEOUT}s
EOF
exit 1
fi
# «Функция» — копируем объект из source в dest
echo "[poc] Вызываем 'функцию': mc cp ${SRC_BUCKET}/${TRIGGER_KEY}${DST_BUCKET}/${TRIGGER_KEY}"
mc cp "${ALIAS}/${SRC_BUCKET}/${TRIGGER_KEY}" "${ALIAS}/${DST_BUCKET}/${TRIGGER_KEY}"
# Проверяем что файл появился в dest
if ! mc ls "${ALIAS}/${DST_BUCKET}/${TRIGGER_KEY}" >/dev/null 2>&1; then
echo "[poc] FAILED: файл не найден в dest bucket" >&2
cat >"$CONFIRM_FILE" <<EOF
FAILED
Копирование в ${DST_BUCKET}/${TRIGGER_KEY} не подтверждено
EOF
exit 1
fi
echo "[poc] SUCCESS: файл скопирован в ${DST_BUCKET}"
cat >"$CONFIRM_FILE" <<EOF
CONFIRMED ✓
Source bucket : ${SRC_BUCKET}
Dest bucket : ${DST_BUCKET}
Object : ${TRIGGER_KEY}
Time : $(date -u)
Что произошло:
1. terraform создал два бакета: ${SRC_BUCKET} и ${DST_BUCKET}
2. Скрипт написал '${TRIGGER_KEY}' в ${SRC_BUCKET}
3. Срабатывание подтверждено через: ${EVENT_METHOD}
4. «Функция» скопировала объект в ${DST_BUCKET}
5. Копия подтверждена — тест пройден
EOF
echo "[poc] apply_confirmation.txt записан"
cat "$CONFIRM_FILE"
@@ -1,32 +0,0 @@
#!/usr/bin/env python3
import datetime
import json
from http.server import BaseHTTPRequestHandler, HTTPServer
LOG_FILE = "function.log"
class Handler(BaseHTTPRequestHandler):
def do_POST(self):
length = int(self.headers.get("Content-Length", 0))
body = self.rfile.read(length).decode("utf-8", errors="ignore")
now = datetime.datetime.utcnow().isoformat() + "Z"
line = f"[{now}] path={self.path} body={body}\n"
with open(LOG_FILE, "a", encoding="utf-8") as f:
f.write(line)
print(line, end="")
self.send_response(200)
self.end_headers()
self.wfile.write(b"ok")
def log_message(self, *_args):
return
if __name__ == "__main__":
server = HTTPServer(("127.0.0.1", 8787), Handler)
print("Function server started at http://127.0.0.1:8787/invoke")
server.serve_forever()
@@ -1,44 +0,0 @@
#!/usr/bin/env python3
import json
import subprocess
import sys
import urllib.request
if len(sys.argv) != 3:
print("usage: poc_watch_and_invoke.py <mc_alias> <bucket>")
sys.exit(1)
alias = sys.argv[1]
bucket = sys.argv[2]
cmd = ["mc", "watch", "--json", f"{alias}/{bucket}"]
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
print(f"Watcher started for {alias}/{bucket}")
for line in proc.stdout:
line = line.strip()
if not line:
continue
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
event = payload.get("eventName", "")
if not event:
continue
data = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
"http://127.0.0.1:8787/invoke",
method="POST",
data=data,
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=5) as r:
_ = r.read()
print(f"forwarded event: {event}")
@@ -1,74 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
if ! command -v mc >/dev/null 2>&1; then
echo "ERROR: mc is not installed"
exit 1
fi
if [[ -z "${S3CFG:-}" ]]; then
echo "ERROR: set S3CFG (path to .s3cfg_registry)"
exit 1
fi
if [[ -z "${BUCKET:-}" ]]; then
echo "ERROR: set BUCKET"
exit 1
fi
RUNTIME_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$RUNTIME_DIR")"
python3 - <<PY > "$PROJECT_DIR/.mc_creds.tmp"
import configparser
cfg=configparser.RawConfigParser(); cfg.read("$S3CFG")
sec=cfg["default"]
print(sec.get("access_key",""))
print(sec.get("secret_key",""))
print(sec.get("host_base",""))
PY
ACCESS_KEY="$(sed -n '1p' "$PROJECT_DIR/.mc_creds.tmp")"
SECRET_KEY="$(sed -n '2p' "$PROJECT_DIR/.mc_creds.tmp")"
HOST_BASE="$(sed -n '3p' "$PROJECT_DIR/.mc_creds.tmp")"
rm -f "$PROJECT_DIR/.mc_creds.tmp"
if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" || -z "$HOST_BASE" ]]; then
echo "ERROR: failed to read credentials from S3CFG"
exit 1
fi
ALIAS="poc"
mc alias rm "$ALIAS" >/dev/null 2>&1 || true
mc alias set "$ALIAS" "https://$HOST_BASE" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null
cd "$PROJECT_DIR"
rm -f function.log .function.pid .watcher.pid
python3 "$RUNTIME_DIR/function_server.py" > function_stdout.log 2>&1 &
echo $! > .function.pid
sleep 1
python3 "$RUNTIME_DIR/poc_watch_and_invoke.py" "$ALIAS" "$BUCKET" > watcher_stdout.log 2>&1 &
echo $! > .watcher.pid
sleep 2
printf "demo-%s\n" "$(date -u +%s)" > /tmp/poc_object.txt
mc cp /tmp/poc_object.txt "$ALIAS/$BUCKET/poc/demo-$(date -u +%s).txt" >/dev/null
sleep 3
echo "=== function.log ==="
if [[ -f function.log ]]; then
cat function.log
else
echo "function.log is empty"
fi
echo "=== watcher_stdout.log (tail) ==="
tail -n 20 watcher_stdout.log || true
echo "Done. To stop background processes:"
WATCHER_PID="$(cat .watcher.pid)"
FUNCTION_PID="$(cat .function.pid)"
echo " kill $WATCHER_PID $FUNCTION_PID"
@@ -1,90 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -lt 2 ]]; then
echo "usage: single_apply_demo.sh <s3cfg_path> <bucket> [timeout_sec]"
exit 1
fi
S3CFG="$1"
BUCKET="$2"
TIMEOUT_SEC="${3:-25}"
WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
ALIAS="applypoc"
cleanup() {
set +e
if [[ -f "$WORKDIR/.watcher.pid" ]]; then kill "$(cat "$WORKDIR/.watcher.pid")" >/dev/null 2>&1 || true; fi
if [[ -f "$WORKDIR/.function.pid" ]]; then kill "$(cat "$WORKDIR/.function.pid")" >/dev/null 2>&1 || true; fi
mc alias rm "$ALIAS" >/dev/null 2>&1 || true
}
trap cleanup EXIT
if ! command -v mc >/dev/null 2>&1; then
echo "FAILED: mc is not installed"
exit 1
fi
if [[ ! -f "$S3CFG" ]]; then
echo "FAILED: S3 cfg not found: $S3CFG"
exit 1
fi
readarray -t CREDS < <(python3 - <<PY
import configparser
cfg=configparser.RawConfigParser(); cfg.read("$S3CFG")
sec=cfg["default"]
print(sec.get("access_key",""))
print(sec.get("secret_key",""))
print(sec.get("host_base",""))
PY
)
ACCESS_KEY="${CREDS[0]:-}"
SECRET_KEY="${CREDS[1]:-}"
HOST_BASE="${CREDS[2]:-}"
if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" || -z "$HOST_BASE" ]]; then
echo "FAILED: invalid credentials in $S3CFG"
exit 1
fi
mc alias set "$ALIAS" "https://$HOST_BASE" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null
cd "$WORKDIR"
rm -f function.log function_stdout.log watcher_stdout.log .function.pid .watcher.pid apply_confirmation.txt
python3 "$WORKDIR/scripts/function_server.py" > "$WORKDIR/function_stdout.log" 2>&1 &
echo $! > "$WORKDIR/.function.pid"
sleep 1
python3 "$WORKDIR/scripts/poc_watch_and_invoke.py" "$ALIAS" "$BUCKET" > "$WORKDIR/watcher_stdout.log" 2>&1 &
echo $! > "$WORKDIR/.watcher.pid"
sleep 2
OBJ="poc/terraform-apply-$(date -u +%s).txt"
printf "poc-%s\n" "$(date -u +%s)" > /tmp/s3_event_poc_payload.txt
mc cp /tmp/s3_event_poc_payload.txt "$ALIAS/$BUCKET/$OBJ" >/dev/null
FOUND=0
for _i in $(seq 1 "$TIMEOUT_SEC"); do
if [[ -s "$WORKDIR/function.log" ]] && grep -q "eventName" "$WORKDIR/function.log"; then
FOUND=1
break
fi
sleep 1
done
if [[ "$FOUND" -eq 1 ]]; then
MSG="CONFIRMED: bucket event reached function for bucket=$BUCKET object=$OBJ"
echo "$MSG"
echo "$MSG" > "$WORKDIR/apply_confirmation.txt"
exit 0
fi
echo "FAILED: no event received by function within ${TIMEOUT_SEC}s"
echo "--- function_stdout.log ---"
tail -n 40 "$WORKDIR/function_stdout.log" || true
echo "--- watcher_stdout.log ---"
tail -n 40 "$WORKDIR/watcher_stdout.log" || true
exit 1
@@ -1,7 +0,0 @@
# Используйте существующий token/s3_uid из:
# TEST_STAND/POSTGRES/terraform.tfvars
api_token = "<token>"
api_endpoint = "https://deck-api-test.ngcloud.ru/api/v1/index.cfm"
s3_uid = "332cdb0d-34bf-43bf-864d-4adcc3b556fb"
bucket_name = "poc-s3-event-demo-001"
@@ -1,33 +0,0 @@
variable "api_token" {
description = "Nubes API token"
type = string
sensitive = true
}
variable "api_endpoint" {
description = "Deck API endpoint"
type = string
default = "https://deck-api-test.ngcloud.ru/api/v1/index.cfm"
}
variable "s3_uid" {
description = "UID существующего nubes_s3 (root S3 service)"
type = string
}
variable "bucket_name" {
description = "Имя создаваемого бакета"
type = string
}
variable "s3cfg_path" {
description = "Путь до .s3cfg_registry для доступа к S3"
type = string
default = "/home/naeel/remote_dev/terraform/secrets/.s3cfg_registry"
}
variable "demo_timeout_sec" {
description = "Таймаут ожидания события в функции"
type = number
default = 25
}
@@ -1,10 +0,0 @@
terraform {
required_version = ">= 1.5.0"
required_providers {
nubes = {
source = "terra.k8c.ru/nubes/nubes"
version = "2.1.23"
}
}
}
+9
View File
@@ -0,0 +1,9 @@
# Terraform state
*.tfstate
*.tfstate.*
.terraform/
.terraform.lock.hcl
# Secrets
terraform.tfvars
*.auto.tfvars
+20
View File
@@ -0,0 +1,20 @@
terraform {
required_providers {
nubes = {
source = "terra.k8c.ru/nubes-test/nubes"
version = "5.0.58"
}
}
}
provider "nubes" {
api_token = var.api_token
api_endpoint = "https://deck-api-test.ngcloud.ru/api/v1/index.cfm"
}
# S3 bucket buck0
resource "nubes_s3bucket" "buck0" {
resource_name = "buck0"
s3_user_uid = var.s3_user_uid
bucket_name = "buck0"
}
@@ -0,0 +1,4 @@
# Токен для TEST — подставь реальный
api_token = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9..."
# S3 realm — платформа развертывания
s3_realm = "s3.msk-1.ngcloud.ru"
+10
View File
@@ -0,0 +1,10 @@
variable "api_token" {
description = "Nubes API token for TEST stand"
type = string
sensitive = true
}
variable "s3_user_uid" {
description = "UUID существующего экземпляра S3 Object Storage"
type = string
}
+6 -1
View File
@@ -828,11 +828,16 @@ func (c *UniversalClient) doRequest(ctx context.Context, method, path string, pa
body = bytes.NewBuffer(b)
}
req, err := http.NewRequestWithContext(ctx, method, c.ApiEndpoint+path, body)
req, err := http.NewRequestWithContext(ctx, method, c.ApiEndpoint, body)
if err != nil {
return nil, nil, err
}
// ?endpoint= формат — как генератор, чтобы пройти DDoS-Guard
q := req.URL.Query()
q.Set("endpoint", path)
req.URL.RawQuery = q.Encode()
req.Close = true
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", userAgent)
+1 -1
View File
@@ -10,7 +10,7 @@ import (
)
var (
version string = "5.0.57"
version string = "5.0.58"
)
func main() {