add: stand configs
This commit is contained in:
“Naeel”
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
if [[ ! -f "$ROOT_DIR/terraform.tfvars" ]]; then
|
||||
echo "ERROR: $ROOT_DIR/terraform.tfvars not found"
|
||||
echo "Create it from terraform.tfvars.example or reuse values from TEST_STAND/POSTGRES/terraform.tfvars"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "$ROOT_DIR"
|
||||
terraform init
|
||||
terraform apply -auto-approve
|
||||
|
||||
BUCKET="$(terraform output -raw bucket_name 2>/dev/null || true)"
|
||||
if [[ -z "$BUCKET" ]]; then
|
||||
echo "ERROR: bucket_name output is empty"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Bucket created: $BUCKET"
|
||||
|
||||
echo "Running event->function demo..."
|
||||
export S3CFG="${S3CFG:-/home/naeel/remote_dev/terraform/secrets/.s3cfg_registry}"
|
||||
export BUCKET
|
||||
bash "$ROOT_DIR/scripts/run_demo.sh"
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env bash
|
||||
# Симуляция event-driven функции:
|
||||
# 1. mc watch следит за SOURCE бакетом (как S3 notification)
|
||||
# 2. При появлении объекта — "функция" копирует его в DEST бакет
|
||||
# 3. Подтверждение записывается в apply_confirmation.txt
|
||||
#
|
||||
# Аргументы: <s3cfg_path> <src_bucket> <dst_bucket> [timeout_sec]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
S3CFG="${1:?s3cfg_path required}"
|
||||
SRC_BUCKET="${2:?src_bucket required}"
|
||||
DST_BUCKET="${3:?dst_bucket required}"
|
||||
TIMEOUT="${4:-35}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
POC_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
CONFIRM_FILE="${POC_DIR}/apply_confirmation.txt"
|
||||
EVENTS_FILE="/tmp/poc_events_$$.json"
|
||||
ALIAS="poc$$"
|
||||
|
||||
cleanup() {
|
||||
kill "$WATCHER_PID" 2>/dev/null || true
|
||||
rm -f "$EVENTS_FILE"
|
||||
mc alias rm "$ALIAS" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "[poc] S3CFG: $S3CFG"
|
||||
echo "[poc] Source bucket : $SRC_BUCKET"
|
||||
echo "[poc] Dest bucket : $DST_BUCKET"
|
||||
echo "[poc] Timeout : ${TIMEOUT}s"
|
||||
|
||||
# Парсим .s3cfg
|
||||
HOST=$(grep -E '^\s*host_base\s*=' "$S3CFG" | head -1 | sed 's/.*=\s*//' | tr -d '[:space:]')
|
||||
AK=$(grep -E '^\s*access_key\s*=' "$S3CFG" | head -1 | sed 's/.*=\s*//' | tr -d '[:space:]')
|
||||
SK=$(grep -E '^\s*secret_key\s*=' "$S3CFG" | head -1 | sed 's/.*=\s*//' | tr -d '[:space:]')
|
||||
USE_HTTPS=$(grep -E '^\s*use_https\s*=' "$S3CFG" | head -1 | sed 's/.*=\s*//' | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
if [[ -z "$HOST" || -z "$AK" || -z "$SK" ]]; then
|
||||
echo "[poc] ERROR: не удалось разобрать .s3cfg" >&2; exit 1
|
||||
fi
|
||||
|
||||
SCHEME="https"
|
||||
if [[ "$USE_HTTPS" == "false" || "$USE_HTTPS" == "0" || "$USE_HTTPS" == "no" ]]; then
|
||||
SCHEME="http"
|
||||
fi
|
||||
|
||||
if [[ "$HOST" == http://* || "$HOST" == https://* ]]; then
|
||||
ENDPOINT="$HOST"
|
||||
else
|
||||
ENDPOINT="${SCHEME}://${HOST}"
|
||||
fi
|
||||
|
||||
# Настраиваем mc alias
|
||||
mc alias set "$ALIAS" "$ENDPOINT" "$AK" "$SK" --api S3v4 >/dev/null 2>&1
|
||||
echo "[poc] mc alias '$ALIAS' → ${ENDPOINT}"
|
||||
|
||||
# Запускаем mc watch на SOURCE (стримит события в файл)
|
||||
touch "$EVENTS_FILE"
|
||||
mc watch --json "${ALIAS}/${SRC_BUCKET}" >"$EVENTS_FILE" 2>&1 &
|
||||
WATCHER_PID=$!
|
||||
echo "[poc] mc watch запущен (pid=$WATCHER_PID)"
|
||||
sleep 2 # даём watch время подключиться
|
||||
|
||||
# Имя триггерного объекта
|
||||
TRIGGER_KEY="event-trigger-$(date +%s).txt"
|
||||
echo "[poc] Пишем объект '${TRIGGER_KEY}' в ${SRC_BUCKET} (trigger)..."
|
||||
echo "S3 event PoC trigger — $(date -u)" | mc pipe "${ALIAS}/${SRC_BUCKET}/${TRIGGER_KEY}"
|
||||
echo "[poc] Объект записан. Ждём события через mc watch (макс ${TIMEOUT}s)..."
|
||||
|
||||
# Ждём появления ключа в потоке событий mc watch
|
||||
ELAPSED=0
|
||||
EVENT_DETECTED=false
|
||||
EVENT_METHOD=""
|
||||
while [[ $ELAPSED -lt $TIMEOUT ]]; do
|
||||
if grep -q "${TRIGGER_KEY}" "$EVENTS_FILE" 2>/dev/null; then
|
||||
EVENT_DETECTED=true
|
||||
EVENT_METHOD="mc watch"
|
||||
echo "[poc] Событие обнаружено в потоке mc watch (через ${ELAPSED}s)"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
ELAPSED=$((ELAPSED + 2))
|
||||
done
|
||||
|
||||
if [[ "$EVENT_DETECTED" != "true" ]]; then
|
||||
echo "[poc] WARN: mc watch не поймал объект за ${TIMEOUT}s, проверяем напрямую..."
|
||||
if mc ls "${ALIAS}/${SRC_BUCKET}/${TRIGGER_KEY}" >/dev/null 2>&1; then
|
||||
EVENT_DETECTED=true
|
||||
EVENT_METHOD="fallback check"
|
||||
echo "[poc] Объект есть в source — выполняем копирование (fallback)"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$EVENT_DETECTED" != "true" ]]; then
|
||||
echo "[poc] FAILED: объект не найден в source bucket" >&2
|
||||
cat >"$CONFIRM_FILE" <<EOF
|
||||
FAILED
|
||||
Объект ${TRIGGER_KEY} не появился в ${SRC_BUCKET} за ${TIMEOUT}s
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# «Функция» — копируем объект из source в dest
|
||||
echo "[poc] Вызываем 'функцию': mc cp ${SRC_BUCKET}/${TRIGGER_KEY} → ${DST_BUCKET}/${TRIGGER_KEY}"
|
||||
mc cp "${ALIAS}/${SRC_BUCKET}/${TRIGGER_KEY}" "${ALIAS}/${DST_BUCKET}/${TRIGGER_KEY}"
|
||||
|
||||
# Проверяем что файл появился в dest
|
||||
if ! mc ls "${ALIAS}/${DST_BUCKET}/${TRIGGER_KEY}" >/dev/null 2>&1; then
|
||||
echo "[poc] FAILED: файл не найден в dest bucket" >&2
|
||||
cat >"$CONFIRM_FILE" <<EOF
|
||||
FAILED
|
||||
Копирование в ${DST_BUCKET}/${TRIGGER_KEY} не подтверждено
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[poc] SUCCESS: файл скопирован в ${DST_BUCKET}"
|
||||
|
||||
cat >"$CONFIRM_FILE" <<EOF
|
||||
CONFIRMED ✓
|
||||
|
||||
Source bucket : ${SRC_BUCKET}
|
||||
Dest bucket : ${DST_BUCKET}
|
||||
Object : ${TRIGGER_KEY}
|
||||
Time : $(date -u)
|
||||
|
||||
Что произошло:
|
||||
1. terraform создал два бакета: ${SRC_BUCKET} и ${DST_BUCKET}
|
||||
2. Скрипт написал '${TRIGGER_KEY}' в ${SRC_BUCKET}
|
||||
3. Срабатывание подтверждено через: ${EVENT_METHOD}
|
||||
4. «Функция» скопировала объект в ${DST_BUCKET}
|
||||
5. Копия подтверждена — тест пройден
|
||||
EOF
|
||||
|
||||
echo "[poc] apply_confirmation.txt записан"
|
||||
cat "$CONFIRM_FILE"
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
import datetime
|
||||
import json
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
LOG_FILE = "function.log"
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length).decode("utf-8", errors="ignore")
|
||||
|
||||
now = datetime.datetime.utcnow().isoformat() + "Z"
|
||||
line = f"[{now}] path={self.path} body={body}\n"
|
||||
|
||||
with open(LOG_FILE, "a", encoding="utf-8") as f:
|
||||
f.write(line)
|
||||
|
||||
print(line, end="")
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(b"ok")
|
||||
|
||||
def log_message(self, *_args):
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
server = HTTPServer(("127.0.0.1", 8787), Handler)
|
||||
print("Function server started at http://127.0.0.1:8787/invoke")
|
||||
server.serve_forever()
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
if len(sys.argv) != 3:
|
||||
print("usage: poc_watch_and_invoke.py <mc_alias> <bucket>")
|
||||
sys.exit(1)
|
||||
|
||||
alias = sys.argv[1]
|
||||
bucket = sys.argv[2]
|
||||
|
||||
cmd = ["mc", "watch", "--json", f"{alias}/{bucket}"]
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
|
||||
print(f"Watcher started for {alias}/{bucket}")
|
||||
|
||||
for line in proc.stdout:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
event = payload.get("eventName", "")
|
||||
if not event:
|
||||
continue
|
||||
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
"http://127.0.0.1:8787/invoke",
|
||||
method="POST",
|
||||
data=data,
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(req, timeout=5) as r:
|
||||
_ = r.read()
|
||||
|
||||
print(f"forwarded event: {event}")
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v mc >/dev/null 2>&1; then
|
||||
echo "ERROR: mc is not installed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${S3CFG:-}" ]]; then
|
||||
echo "ERROR: set S3CFG (path to .s3cfg_registry)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "${BUCKET:-}" ]]; then
|
||||
echo "ERROR: set BUCKET"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RUNTIME_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$RUNTIME_DIR")"
|
||||
|
||||
python3 - <<PY > "$PROJECT_DIR/.mc_creds.tmp"
|
||||
import configparser
|
||||
cfg=configparser.RawConfigParser(); cfg.read("$S3CFG")
|
||||
sec=cfg["default"]
|
||||
print(sec.get("access_key",""))
|
||||
print(sec.get("secret_key",""))
|
||||
print(sec.get("host_base",""))
|
||||
PY
|
||||
|
||||
ACCESS_KEY="$(sed -n '1p' "$PROJECT_DIR/.mc_creds.tmp")"
|
||||
SECRET_KEY="$(sed -n '2p' "$PROJECT_DIR/.mc_creds.tmp")"
|
||||
HOST_BASE="$(sed -n '3p' "$PROJECT_DIR/.mc_creds.tmp")"
|
||||
rm -f "$PROJECT_DIR/.mc_creds.tmp"
|
||||
|
||||
if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" || -z "$HOST_BASE" ]]; then
|
||||
echo "ERROR: failed to read credentials from S3CFG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ALIAS="poc"
|
||||
mc alias rm "$ALIAS" >/dev/null 2>&1 || true
|
||||
mc alias set "$ALIAS" "https://$HOST_BASE" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
rm -f function.log .function.pid .watcher.pid
|
||||
|
||||
python3 "$RUNTIME_DIR/function_server.py" > function_stdout.log 2>&1 &
|
||||
echo $! > .function.pid
|
||||
|
||||
sleep 1
|
||||
python3 "$RUNTIME_DIR/poc_watch_and_invoke.py" "$ALIAS" "$BUCKET" > watcher_stdout.log 2>&1 &
|
||||
echo $! > .watcher.pid
|
||||
|
||||
sleep 2
|
||||
printf "demo-%s\n" "$(date -u +%s)" > /tmp/poc_object.txt
|
||||
mc cp /tmp/poc_object.txt "$ALIAS/$BUCKET/poc/demo-$(date -u +%s).txt" >/dev/null
|
||||
|
||||
sleep 3
|
||||
|
||||
echo "=== function.log ==="
|
||||
if [[ -f function.log ]]; then
|
||||
cat function.log
|
||||
else
|
||||
echo "function.log is empty"
|
||||
fi
|
||||
|
||||
echo "=== watcher_stdout.log (tail) ==="
|
||||
tail -n 20 watcher_stdout.log || true
|
||||
|
||||
echo "Done. To stop background processes:"
|
||||
WATCHER_PID="$(cat .watcher.pid)"
|
||||
FUNCTION_PID="$(cat .function.pid)"
|
||||
echo " kill $WATCHER_PID $FUNCTION_PID"
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $# -lt 2 ]]; then
|
||||
echo "usage: single_apply_demo.sh <s3cfg_path> <bucket> [timeout_sec]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
S3CFG="$1"
|
||||
BUCKET="$2"
|
||||
TIMEOUT_SEC="${3:-25}"
|
||||
|
||||
WORKDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
ALIAS="applypoc"
|
||||
|
||||
cleanup() {
|
||||
set +e
|
||||
if [[ -f "$WORKDIR/.watcher.pid" ]]; then kill "$(cat "$WORKDIR/.watcher.pid")" >/dev/null 2>&1 || true; fi
|
||||
if [[ -f "$WORKDIR/.function.pid" ]]; then kill "$(cat "$WORKDIR/.function.pid")" >/dev/null 2>&1 || true; fi
|
||||
mc alias rm "$ALIAS" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
if ! command -v mc >/dev/null 2>&1; then
|
||||
echo "FAILED: mc is not installed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$S3CFG" ]]; then
|
||||
echo "FAILED: S3 cfg not found: $S3CFG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
readarray -t CREDS < <(python3 - <<PY
|
||||
import configparser
|
||||
cfg=configparser.RawConfigParser(); cfg.read("$S3CFG")
|
||||
sec=cfg["default"]
|
||||
print(sec.get("access_key",""))
|
||||
print(sec.get("secret_key",""))
|
||||
print(sec.get("host_base",""))
|
||||
PY
|
||||
)
|
||||
ACCESS_KEY="${CREDS[0]:-}"
|
||||
SECRET_KEY="${CREDS[1]:-}"
|
||||
HOST_BASE="${CREDS[2]:-}"
|
||||
|
||||
if [[ -z "$ACCESS_KEY" || -z "$SECRET_KEY" || -z "$HOST_BASE" ]]; then
|
||||
echo "FAILED: invalid credentials in $S3CFG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mc alias set "$ALIAS" "https://$HOST_BASE" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null
|
||||
|
||||
cd "$WORKDIR"
|
||||
rm -f function.log function_stdout.log watcher_stdout.log .function.pid .watcher.pid apply_confirmation.txt
|
||||
|
||||
python3 "$WORKDIR/scripts/function_server.py" > "$WORKDIR/function_stdout.log" 2>&1 &
|
||||
echo $! > "$WORKDIR/.function.pid"
|
||||
|
||||
sleep 1
|
||||
python3 "$WORKDIR/scripts/poc_watch_and_invoke.py" "$ALIAS" "$BUCKET" > "$WORKDIR/watcher_stdout.log" 2>&1 &
|
||||
echo $! > "$WORKDIR/.watcher.pid"
|
||||
|
||||
sleep 2
|
||||
OBJ="poc/terraform-apply-$(date -u +%s).txt"
|
||||
printf "poc-%s\n" "$(date -u +%s)" > /tmp/s3_event_poc_payload.txt
|
||||
mc cp /tmp/s3_event_poc_payload.txt "$ALIAS/$BUCKET/$OBJ" >/dev/null
|
||||
|
||||
FOUND=0
|
||||
for _i in $(seq 1 "$TIMEOUT_SEC"); do
|
||||
if [[ -s "$WORKDIR/function.log" ]] && grep -q "eventName" "$WORKDIR/function.log"; then
|
||||
FOUND=1
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [[ "$FOUND" -eq 1 ]]; then
|
||||
MSG="CONFIRMED: bucket event reached function for bucket=$BUCKET object=$OBJ"
|
||||
echo "$MSG"
|
||||
echo "$MSG" > "$WORKDIR/apply_confirmation.txt"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "FAILED: no event received by function within ${TIMEOUT_SEC}s"
|
||||
echo "--- function_stdout.log ---"
|
||||
tail -n 40 "$WORKDIR/function_stdout.log" || true
|
||||
echo "--- watcher_stdout.log ---"
|
||||
tail -n 40 "$WORKDIR/watcher_stdout.log" || true
|
||||
exit 1
|
||||
Reference in New Issue
Block a user