From f11902e81b919049502efe45e7f37ad07d1087d5 Mon Sep 17 00:00:00 2001 From: Sanket Sudake Date: Fri, 25 Nov 2022 15:48:09 +0530 Subject: [PATCH] Backup prometheus data from CI run (#2636) * Backup Prometheus data from CI run * use `kube-prometheus-stack` instead of `prometheus` * Add dump-analyzer script * Fix monitor helm chart values * Update readme for dump-analyzer * Disable grafana and alertmanager Signed-off-by: Sanket Sudake --- .github/workflows/push_pr.yaml | 18 ++- .gitignore | 3 + cmd/fission-bundle/main.go | 2 +- hack/backup-prometheus.sh | 23 ++++ hack/dump-analyzer | 199 +++++++++++++++++++++++++++++++++ hack/dump-analyzer.md | 133 ++++++++++++++++++++++ skaffold.yaml | 18 ++- 7 files changed, 393 insertions(+), 3 deletions(-) create mode 100755 hack/backup-prometheus.sh create mode 100755 hack/dump-analyzer create mode 100644 hack/dump-analyzer.md diff --git a/.github/workflows/push_pr.yaml b/.github/workflows/push_pr.yaml index 10922c53..ffe3d6a3 100644 --- a/.github/workflows/push_pr.yaml +++ b/.github/workflows/push_pr.yaml @@ -91,7 +91,8 @@ jobs: helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create ns monitoring - helm install monitoring prometheus-community/prometheus -n monitoring + helm install prometheus prometheus-community/kube-prometheus-stack -n monitoring \ + --set grafana.enabled=false --set alertmanager.enabled=false - name: Build and Install Fission CLI run: | @@ -133,6 +134,12 @@ jobs: run: | kind export logs --name kind kind-logs + - name: Backup prometheus data + timeout-minutes: 10 + if: ${{ always() }} + run: | + TRACE=1 ./hack/backup-prometheus.sh + - name: Archive fission dump timeout-minutes: 10 if: ${{ failure() || cancelled() }} @@ -142,6 +149,15 @@ jobs: path: fission-dump/*.zip retention-days: 5 + - name: Archive prometheus dump + timeout-minutes: 10 + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: prom-dump-${{ github.run_id }}-${{ matrix.kindversion }} + path: /tmp/prometheus/* + retention-days: 5 + - name: Archive kind logs timeout-minutes: 10 if: ${{ always() }} diff --git a/.gitignore b/.gitignore index d0ee03d9..73852f2d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ manifest/ coverage.txt cosign.key + +# Dumps +.dumps/ diff --git a/cmd/fission-bundle/main.go b/cmd/fission-bundle/main.go index 1d345d27..3bf18e35 100644 --- a/cmd/fission-bundle/main.go +++ b/cmd/fission-bundle/main.go @@ -216,7 +216,7 @@ Options: ctx := signals.SetupSignalHandler() profile.ProfileIfEnabled(ctx, logger) - version := fmt.Sprintf("Fission Bundle Version: %v", info.BuildInfo().String()) + version := fmt.Sprintf("Fission Bundle Version: %s", info.BuildInfo().String()) arguments, err := docopt.ParseArgs(usage, nil, version) if err != nil { logger.Error("failed to parse arguments", zap.Error(err)) diff --git a/hack/backup-prometheus.sh b/hack/backup-prometheus.sh new file mode 100755 index 00000000..b3abfe8c --- /dev/null +++ b/hack/backup-prometheus.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +if [[ -n "${TRACE-}" ]]; then + set -o xtrace +fi +NAMESPACE=${1:-monitoring} +RELEASE=${2:-prometheus} +BACKUP_DIR=${3:-/tmp/prometheus} +LABELS="-l app=kube-prometheus-stack-${RELEASE}" + +sts_name=$(kubectl -n "$NAMESPACE" get sts "$LABELS" -o custom-columns=:.metadata.name | grep ^"$RELEASE") +echo "Statefulset name: $sts_name" + +mkdir -p "$BACKUP_DIR"/prometheus +# Copy snapshot to local +kubectl -n "$NAMESPACE" cp "$sts_name"-0:/prometheus/ "$BACKUP_DIR"/prometheus + +mkdir -p "$BACKUP_DIR"/etc/prometheus +kubectl -n "$NAMESPACE" cp "$sts_name"-0:/etc/prometheus "$BACKUP_DIR"/etc/prometheus diff --git a/hack/dump-analyzer b/hack/dump-analyzer new file mode 100755 index 00000000..1b29410d --- /dev/null +++ b/hack/dump-analyzer @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +### +# This script is used to analyze the dump files generated by the Fission CI. +### +set -o errexit +set -o nounset +set -o pipefail +if [[ -n "${TRACE-}" ]]; then + set -o xtrace +fi +REPO="fission/fission" + +function exit_error() { + echo "$1" + exit 1 +} + +function doit() { + echo "! $*" + "$@" +} + +function check_context() { + if [ -z "$DUMP_CONTEXT" ]; then + exit_error "DUMP_CONTEXT not set" + fi +} + +function prom_context() { + if [ -z "$PROM_CONTEXT" ]; then + exit_error "PROM_CONTEXT not set" + fi +} + +function list_context() { + for dump in $(find .dumps/$RUN_ID/extract -type d -depth 1); do + echo "==== $dump ====" + echo "-- Fission Version --" + cat $dump/fission-version/fission-version.txt | grep "server:\|client\:\|Version" + echo "-- K8s Version --" + cat $dump/kubernetes-version/kubernetes-version.txt | grep gitVersion + done +} + +function list_kind() { + for dump in $(find .dumps/$RUN_ID -type d -depth 1 -name 'kind-logs-*'); do + echo "==== $dump ====" + cat $dump/kind-version.txt + echo -e "\n" + done +} + +function extract() { + ARTIFACT_PATH=.dumps/$RUN_ID + if [ ! -d "$ARTIFACT_PATH" ]; then + doit mkdir -p "$ARTIFACT_PATH" + fi + echo $ARTIFACT_PATH + # Don't download if artifact already exits for run + if [ ! -d "$ARTIFACT_PATH"/fission-dump/ ]; then + doit gh run download "$RUN_ID" -R "$REPO" -D "$ARTIFACT_PATH" + fi + if [ -d "$ARTIFACT_PATH"/extract ]; then + doit rm -r "$ARTIFACT_PATH"/extract + fi + DUMPS=$(find "$ARTIFACT_PATH" -name 'fission-dump*.zip') + for dump in $DUMPS; do + doit unzip -q $dump -d "$ARTIFACT_PATH"/extract + done +} + +function info() { + check_context + echo "Dump context: $DUMP_CONTEXT" + doit cat "$DUMP_CONTEXT"/fission-version/fission-version.txt + doit cat "$DUMP_CONTEXT"/kubernetes-version/kubernetes-version.txt +} + +function race_conditions() { + check_context + # doit ack -A 10 "WARNING: DATA RACE" "$DUMP_CONTEXT/" + LOG_FILES=$(find $DUMP_CONTEXT -name '*.txt' -type f | grep log) + for logfile in $LOG_FILES; do + out=$(sed -n '/DATA RACE/,/==================$/p' "$logfile") + if [ "$out" != "" ]; then + shortname=$(basename $logfile) + echo "Trace from $shortname" + sed -n '/DATA RACE/,/==================$/p' "$logfile" + fi + done +} + +function error_logs() { + check_context + # ack '"level":"error"' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | go-slearch -pts,caller,msg,error,logger,errorVerbose | sort + ack -i 'error' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | sort +} + +function prometheus_run() { + # check if prometheus binary exists + if command -v prometheus &> /dev/null ; then + echo "prometheus binary found" + else + exit_error "prometheus binary not found" + fi + prom_context + TSDB_PATH=$PROM_CONTEXT/prometheus + ETC_PROM=$PROM_CONTEXT/etc/prometheus + prometheus \ + --web.console.templates=$ETC_PROM/consoles \ + --web.console.libraries=$ETC_PROM/console_libraries \ + --storage.tsdb.retention.time=10d \ + --config.file=$ETC_PROM/config_out/prometheus.env.yaml \ + --storage.tsdb.path=${TSDB_PATH}/ \ + --web.enable-lifecycle \ + --web.listen-address=127.0.0.1:9090 \ + --web.route-prefix=/ \ + --storage.tsdb.wal-compression \ + --web.config.file=$ETC_PROM/web_config/web-config.yaml +} + +function usage { + echo "./$(basename "$0") [OPTIONS]" + echo "Utilities related to fission dump analysis" + echo " +Options: + -h Show usage + -x [run_id] Download and extract dump locally + -l [run_id] List dump files for a run + -k [run_id] List kind exports in a run + + Following options required DUMP_CONTEXT variable set. + + -i Display dump info + -r Find all race conditions in dump + -e Find all errors in logs + + Following options require PROM_CONTEXT to be set. + + -p Run prometheus with the dump" + exit 3 +} + +# list of arguments expected in the input +optstring=":hxirlekp" + +if [[ ${#} -eq 0 ]]; then + usage +fi + +while getopts ${optstring} arg; do + case ${arg} in + h) + echo "showing usage!" + usage + ;; + x) + RUN_ID=$2 + if [ -z "$RUN_ID" ]; then + exit_error "run id not mentioned" + fi + extract + ;; + l) + RUN_ID=$2 + if [ -z "$RUN_ID" ]; then + exit_error "run id not mentioned" + fi + list_context + ;; + i) + info + ;; + r) + race_conditions + ;; + e) + error_logs + ;; + k) + RUN_ID=$2 + if [ -z "$RUN_ID" ]; then + exit_error "run id not mentioned" + fi + list_kind + ;; + p) + prometheus_run + ;; + :) + echo "$0: Must supply an argument to -$OPTARG." >&2 + exit 1 + ;; + ?) + echo "Invalid option: -${OPTARG}." + exit 2 + ;; + esac +done diff --git a/hack/dump-analyzer.md b/hack/dump-analyzer.md new file mode 100644 index 00000000..f61bb039 --- /dev/null +++ b/hack/dump-analyzer.md @@ -0,0 +1,133 @@ +# Dump analyzer + +## Usage + +```text +./hack/dump-analyzer -h +showing usage! +./dump-analyzer [OPTIONS] +Utilities related to fission dump analysis + +Options: + -h Show usage + -x [run_id] Download and extract dump locally + -l [run_id] List dump files for a run + -k [run_id] List kind exports in a run + + Following options required DUMP_CONTEXT variable set. + + -i Display dump info + -r Find all race conditions in dump + -e Find all errors in logs + + Following options require PROM_CONTEXT to be set. + + -p Run prometheus with the dump + +``` + +## Downloading dump + +```sh +$ ./hack/dump-analyzer -x 3179903728 +! mkdir -p .dumps/3179903728 +.dumps/3179903728 +! gh run download 3179903728 -R fission/fission -D .dumps/3179903728 +! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.19.16/fission-dump_1664866624.zip -d .dumps/3179903728/extract +! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.21.12/fission-dump_1664866628.zip -d .dumps/3179903728/extract +! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.20.15/fission-dump_1664866662.zip -d .dumps/3179903728/extract +``` + +## Check all dumps + +```sh +./hack/dump-analyzer -l 3179903728 +==== .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304 ==== +-- Fission Version -- +client: + Version: v0.0.0 +server: + Version: v0.0.0 +-- K8s Version -- +gitVersion: v1.21.12 +==== .dumps/3179903728/extract/e7febf06-79ce-4eb7-bba0-3fc66aa4bf542096519315 ==== +-- Fission Version -- +client: + Version: v0.0.0 +server: + Version: v0.0.0 +-- K8s Version -- +gitVersion: v1.19.16 +==== .dumps/3179903728/extract/3b339dc2-e9dd-4f7a-a721-41e16d9d248f4152710428 ==== +-- Fission Version -- +client: + Version: v0.0.0 +server: + Version: v0.0.0 +-- K8s Version -- +gitVersion: v1.20.15 +``` + +We can see that the dump is from 3 different versions of k8s. +To explore specific dump set `DUMP_CONTEXT` environment variable. + +```sh +$ export DUMP_CONTEXT=.dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304 + +$ ./hack/dump-analyzer -i +Dump context: .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304 +! cat .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304/fission-version/fission-version.txt +client: + fission/core: + BuildDate: "2022-10-04T06:28:54Z" + GitCommit: 9e74b01 + Version: v0.0.0 +server: + fission/core: + BuildDate: "2022-10-04T06:29:01Z" + GitCommit: 9e74b01 + Version: v0.0.0 +! cat .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304/kubernetes-version/kubernetes-version.txt +buildDate: "2022-05-19T20:02:29Z" +compiler: gc +gitCommit: 696a9fdd2a58340e61e0d815c5769d266fca0802 +gitTreeState: clean +gitVersion: v1.21.12 +goVersion: go1.16.15 +major: "1" +minor: "21" +platform: linux/amd64 + +# See all errors in the dump +$ ./hack/dump-analyzer -e + +# If you want to see error in specific dump you can also grep + +$ grep -rin "string" $DUMP_CONTEXT + +# OR + +$ ack "string" $DUMP_CONTEXT +``` + +## Kind logs + +```sh + ./hack/dump-analyzer -k 3179903728 +==== .dumps/3179903728/kind-logs-3179903728-v1.20.15 ==== +kind v0.14.0 go1.18.2 linux/amd64 + +==== .dumps/3179903728/kind-logs-3179903728-v1.19.16 ==== +kind v0.14.0 go1.18.2 linux/amd64 + +==== .dumps/3179903728/kind-logs-3179903728-v1.21.12 ==== +kind v0.14.0 go1.18.2 linux/amd64 +``` + +## Prometheus with dump + +```sh +$ export DUMP_CONTEXT=.dumps/3539792658/prom-dump-3539792658-v1.21.14 +$ ./hack/dump-analyzer -p +# Visit localhost:9090 +``` diff --git a/skaffold.yaml b/skaffold.yaml index cb9b6b60..c4d737bd 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -66,6 +66,13 @@ deploy: webhook.securityContext.enabled: true storagesvc.securityContext.enabled: true controller.enabled: false + serviceMonitor.enabled: false + serviceMonitor.namespace: monitoring + serviceMonitor.additionalServiceMonitorLabels.release: prometheus + podMonitor.enabled: false + podMonitor.namespace: monitoring + podMonitor.additionalPodMonitorLabels.release: prometheus + grafana.dashboards.enabled: false wait: true flags: install: @@ -118,7 +125,16 @@ profiles: value: true - op: replace path: /deploy/helm/releases/0/setValues/prometheus.serviceEndpoint - value: "http://monitoring-prometheus-server.monitoring.svc.cluster.local" + value: "http://prometheus-operated.monitoring.svc.cluster.local:9090" + - op: replace + path: /deploy/helm/releases/0/setValues/podMonitor.enabled + value: true + - op: replace + path: /deploy/helm/releases/0/setValues/serviceMonitor.enabled + value: true + - op: replace + path: /deploy/helm/releases/0/setValues/grafana.dashboards.enabled + value: true - name: kind-opentelemetry patches: - op: replace