Backup prometheus data from CI run (#2636)

* Backup Prometheus data from CI run
* use `kube-prometheus-stack` instead of `prometheus`
* Add dump-analyzer script
* Fix monitor helm chart values
* Update readme for dump-analyzer
* Disable grafana and alertmanager

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

.github/workflows/push_pr.yaml

@@ -91,7 +91,8 @@ jobs:
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo update
           kubectl create ns monitoring
-          helm install monitoring prometheus-community/prometheus -n monitoring
+          helm install prometheus prometheus-community/kube-prometheus-stack -n monitoring \
+          --set grafana.enabled=false --set alertmanager.enabled=false
 
       - name: Build and Install Fission CLI
         run: |
@@ -133,6 +134,12 @@ jobs:
         run: |
           kind export logs --name kind kind-logs
 
+      - name: Backup prometheus data
+        timeout-minutes: 10
+        if: ${{ always() }}
+        run: |
+          TRACE=1 ./hack/backup-prometheus.sh
+
       - name: Archive fission dump
         timeout-minutes: 10
         if: ${{ failure() || cancelled() }}
@@ -142,6 +149,15 @@ jobs:
           path: fission-dump/*.zip
           retention-days: 5
 
+      - name: Archive prometheus dump
+        timeout-minutes: 10
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: prom-dump-${{ github.run_id }}-${{ matrix.kindversion }}
+          path: /tmp/prometheus/*
+          retention-days: 5
+
       - name: Archive kind logs
         timeout-minutes: 10
         if: ${{ always() }}

.gitignore

@@ -37,3 +37,6 @@ manifest/
 coverage.txt
 
 cosign.key
+
+# Dumps
+.dumps/

cmd/fission-bundle/main.go

@@ -216,7 +216,7 @@ Options:
 	ctx := signals.SetupSignalHandler()
 	profile.ProfileIfEnabled(ctx, logger)
 
-	version := fmt.Sprintf("Fission Bundle Version: %v", info.BuildInfo().String())
+	version := fmt.Sprintf("Fission Bundle Version: %s", info.BuildInfo().String())
 	arguments, err := docopt.ParseArgs(usage, nil, version)
 	if err != nil {
 		logger.Error("failed to parse arguments", zap.Error(err))

hack/backup-prometheus.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ -n "${TRACE-}" ]]; then
+    set -o xtrace
+fi
+NAMESPACE=${1:-monitoring}
+RELEASE=${2:-prometheus}
+BACKUP_DIR=${3:-/tmp/prometheus}
+LABELS="-l app=kube-prometheus-stack-${RELEASE}"
+
+sts_name=$(kubectl -n "$NAMESPACE" get sts "$LABELS" -o custom-columns=:.metadata.name | grep ^"$RELEASE")
+echo "Statefulset name: $sts_name"
+
+mkdir -p "$BACKUP_DIR"/prometheus
+# Copy snapshot to local
+kubectl -n "$NAMESPACE" cp "$sts_name"-0:/prometheus/ "$BACKUP_DIR"/prometheus
+
+mkdir -p "$BACKUP_DIR"/etc/prometheus
+kubectl -n "$NAMESPACE" cp "$sts_name"-0:/etc/prometheus "$BACKUP_DIR"/etc/prometheus

hack/dump-analyzer

@@ -0,0 +1,199 @@
+#!/usr/bin/env bash
+###
+# This script is used to analyze the dump files generated by the Fission CI.
+###
+set -o errexit
+set -o nounset
+set -o pipefail
+if [[ -n "${TRACE-}" ]]; then
+    set -o xtrace
+fi
+REPO="fission/fission"
+
+function exit_error() {
+    echo "$1"
+    exit 1
+}
+
+function doit() {
+    echo "! $*"
+    "$@"
+}
+
+function check_context() {
+    if [ -z "$DUMP_CONTEXT" ]; then
+        exit_error "DUMP_CONTEXT not set"
+    fi
+}
+
+function prom_context() {
+    if [ -z "$PROM_CONTEXT" ]; then
+        exit_error "PROM_CONTEXT not set"
+    fi
+}
+
+function list_context() {
+    for dump in $(find .dumps/$RUN_ID/extract -type d -depth 1); do
+        echo "==== $dump ===="
+        echo "-- Fission Version --"
+        cat $dump/fission-version/fission-version.txt | grep "server:\|client\:\|Version"
+        echo "-- K8s Version --"
+        cat $dump/kubernetes-version/kubernetes-version.txt | grep gitVersion
+    done
+}
+
+function list_kind() {
+    for dump in $(find .dumps/$RUN_ID -type d -depth 1 -name 'kind-logs-*'); do
+        echo "==== $dump ===="
+        cat $dump/kind-version.txt
+        echo -e "\n"
+    done
+}
+
+function extract() {
+    ARTIFACT_PATH=.dumps/$RUN_ID
+    if [ ! -d "$ARTIFACT_PATH" ]; then
+        doit mkdir -p "$ARTIFACT_PATH"
+    fi
+    echo $ARTIFACT_PATH
+    # Don't download if artifact already exits for run
+    if [ ! -d "$ARTIFACT_PATH"/fission-dump/ ]; then
+        doit gh run download "$RUN_ID" -R "$REPO" -D "$ARTIFACT_PATH"
+    fi
+    if [ -d "$ARTIFACT_PATH"/extract ]; then
+        doit rm -r "$ARTIFACT_PATH"/extract
+    fi
+    DUMPS=$(find "$ARTIFACT_PATH" -name 'fission-dump*.zip')
+    for dump in $DUMPS; do
+        doit unzip -q $dump -d "$ARTIFACT_PATH"/extract
+    done
+}
+
+function info() {
+    check_context
+    echo "Dump context: $DUMP_CONTEXT"
+    doit cat "$DUMP_CONTEXT"/fission-version/fission-version.txt
+    doit cat "$DUMP_CONTEXT"/kubernetes-version/kubernetes-version.txt
+}
+
+function race_conditions() {
+    check_context
+    # doit ack -A 10 "WARNING: DATA RACE" "$DUMP_CONTEXT/"
+    LOG_FILES=$(find $DUMP_CONTEXT -name '*.txt' -type f | grep log)
+    for logfile in $LOG_FILES; do
+        out=$(sed -n '/DATA RACE/,/==================$/p' "$logfile")
+        if [ "$out" != "" ]; then
+            shortname=$(basename $logfile)
+            echo "Trace from $shortname"
+            sed -n '/DATA RACE/,/==================$/p' "$logfile"
+        fi
+    done
+}
+
+function error_logs() {
+    check_context
+    # ack '"level":"error"' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | go-slearch -pts,caller,msg,error,logger,errorVerbose | sort
+    ack -i 'error' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | sort
+}
+
+function prometheus_run() {
+    # check if prometheus binary exists
+    if command -v prometheus &> /dev/null ; then
+        echo "prometheus binary found"
+    else
+        exit_error "prometheus binary not found"
+    fi
+    prom_context
+    TSDB_PATH=$PROM_CONTEXT/prometheus
+    ETC_PROM=$PROM_CONTEXT/etc/prometheus
+    prometheus \
+        --web.console.templates=$ETC_PROM/consoles \
+        --web.console.libraries=$ETC_PROM/console_libraries \
+        --storage.tsdb.retention.time=10d \
+        --config.file=$ETC_PROM/config_out/prometheus.env.yaml \
+        --storage.tsdb.path=${TSDB_PATH}/ \
+        --web.enable-lifecycle \
+        --web.listen-address=127.0.0.1:9090 \
+        --web.route-prefix=/ \
+        --storage.tsdb.wal-compression \
+        --web.config.file=$ETC_PROM/web_config/web-config.yaml
+}
+
+function usage {
+    echo "./$(basename "$0") [OPTIONS]"
+    echo "Utilities related to fission dump analysis"
+    echo "
+Options:
+    -h              Show usage
+    -x [run_id]     Download and extract dump locally
+    -l [run_id]    List dump files for a run
+    -k [run_id]    List kind exports in a run
+
+    Following options required DUMP_CONTEXT variable set.
+
+    -i              Display dump info
+    -r              Find all race conditions in dump
+    -e              Find all errors in logs
+
+    Following options require PROM_CONTEXT to be set.
+
+    -p             Run prometheus with the dump"
+    exit 3
+}
+
+# list of arguments expected in the input
+optstring=":hxirlekp"
+
+if [[ ${#} -eq 0 ]]; then
+    usage
+fi
+
+while getopts ${optstring} arg; do
+    case ${arg} in
+    h)
+        echo "showing usage!"
+        usage
+        ;;
+    x)
+        RUN_ID=$2
+        if [ -z "$RUN_ID" ]; then
+            exit_error "run id not mentioned"
+        fi
+        extract
+        ;;
+    l)
+        RUN_ID=$2
+        if [ -z "$RUN_ID" ]; then
+            exit_error "run id not mentioned"
+        fi
+        list_context
+        ;;
+    i)
+        info
+        ;;
+    r)
+        race_conditions
+        ;;
+    e)
+        error_logs
+        ;;
+    k)
+        RUN_ID=$2
+        if [ -z "$RUN_ID" ]; then
+            exit_error "run id not mentioned"
+        fi
+        list_kind
+        ;;
+    p)
+        prometheus_run
+        ;;
+    :)
+        echo "$0: Must supply an argument to -$OPTARG." >&2
+        exit 1
+        ;;
+    ?)
+        echo "Invalid option: -${OPTARG}."
+        exit 2
+        ;;
+    esac
+done

hack/dump-analyzer.md

@@ -0,0 +1,133 @@
+# Dump analyzer
+
+## Usage
+
+```text
+./hack/dump-analyzer -h
+showing usage!
+./dump-analyzer [OPTIONS]
+Utilities related to fission dump analysis
+
+Options:
+    -h              Show usage
+    -x [run_id]     Download and extract dump locally
+    -l [run_id]    List dump files for a run
+    -k [run_id]    List kind exports in a run
+
+    Following options required DUMP_CONTEXT variable set.
+
+    -i              Display dump info
+    -r              Find all race conditions in dump
+    -e              Find all errors in logs
+
+    Following options require PROM_CONTEXT to be set.
+
+    -p             Run prometheus with the dump
+
+```
+
+## Downloading dump
+
+```sh
+$ ./hack/dump-analyzer -x 3179903728
+! mkdir -p .dumps/3179903728
+.dumps/3179903728
+! gh run download 3179903728 -R fission/fission -D .dumps/3179903728
+! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.19.16/fission-dump_1664866624.zip -d .dumps/3179903728/extract
+! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.21.12/fission-dump_1664866628.zip -d .dumps/3179903728/extract
+! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.20.15/fission-dump_1664866662.zip -d .dumps/3179903728/extract
+```
+
+## Check all dumps
+
+```sh
+./hack/dump-analyzer -l 3179903728
+==== .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304 ====
+-- Fission Version --
+client:
+    Version: v0.0.0
+server:
+    Version: v0.0.0
+-- K8s Version --
+gitVersion: v1.21.12
+==== .dumps/3179903728/extract/e7febf06-79ce-4eb7-bba0-3fc66aa4bf542096519315 ====
+-- Fission Version --
+client:
+    Version: v0.0.0
+server:
+    Version: v0.0.0
+-- K8s Version --
+gitVersion: v1.19.16
+==== .dumps/3179903728/extract/3b339dc2-e9dd-4f7a-a721-41e16d9d248f4152710428 ====
+-- Fission Version --
+client:
+    Version: v0.0.0
+server:
+    Version: v0.0.0
+-- K8s Version --
+gitVersion: v1.20.15
+```
+
+We can see that the dump is from 3 different versions of k8s.
+To explore specific dump set `DUMP_CONTEXT` environment variable.
+
+```sh
+$ export DUMP_CONTEXT=.dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304
+
+$ ./hack/dump-analyzer -i
+Dump context: .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304
+! cat .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304/fission-version/fission-version.txt
+client:
+  fission/core:
+    BuildDate: "2022-10-04T06:28:54Z"
+    GitCommit: 9e74b01
+    Version: v0.0.0
+server:
+  fission/core:
+    BuildDate: "2022-10-04T06:29:01Z"
+    GitCommit: 9e74b01
+    Version: v0.0.0
+! cat .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304/kubernetes-version/kubernetes-version.txt
+buildDate: "2022-05-19T20:02:29Z"
+compiler: gc
+gitCommit: 696a9fdd2a58340e61e0d815c5769d266fca0802
+gitTreeState: clean
+gitVersion: v1.21.12
+goVersion: go1.16.15
+major: "1"
+minor: "21"
+platform: linux/amd64
+
+# See all errors in the dump
+$ ./hack/dump-analyzer -e
+
+# If you want to see error in specific dump you can also grep
+
+$ grep -rin "string" $DUMP_CONTEXT
+
+# OR
+
+$ ack "string" $DUMP_CONTEXT
+```
+
+## Kind logs
+
+```sh
+ ./hack/dump-analyzer -k 3179903728
+==== .dumps/3179903728/kind-logs-3179903728-v1.20.15 ====
+kind v0.14.0 go1.18.2 linux/amd64
+
+==== .dumps/3179903728/kind-logs-3179903728-v1.19.16 ====
+kind v0.14.0 go1.18.2 linux/amd64
+
+==== .dumps/3179903728/kind-logs-3179903728-v1.21.12 ====
+kind v0.14.0 go1.18.2 linux/amd64
+```
+
+## Prometheus with dump
+
+```sh
+$ export DUMP_CONTEXT=.dumps/3539792658/prom-dump-3539792658-v1.21.14
+$ ./hack/dump-analyzer -p
+# Visit localhost:9090
+```

skaffold.yaml

@@ -66,6 +66,13 @@ deploy:
         webhook.securityContext.enabled: true
         storagesvc.securityContext.enabled: true
         controller.enabled: false
+        serviceMonitor.enabled: false
+        serviceMonitor.namespace: monitoring
+        serviceMonitor.additionalServiceMonitorLabels.release: prometheus
+        podMonitor.enabled: false
+        podMonitor.namespace: monitoring
+        podMonitor.additionalPodMonitorLabels.release: prometheus
+        grafana.dashboards.enabled: false
       wait: true
     flags:
       install:
@@ -118,7 +125,16 @@ profiles:
     value: true
   - op: replace
     path: /deploy/helm/releases/0/setValues/prometheus.serviceEndpoint
-    value: "http://monitoring-prometheus-server.monitoring.svc.cluster.local"
+    value: "http://prometheus-operated.monitoring.svc.cluster.local:9090"
+  - op: replace
+    path: /deploy/helm/releases/0/setValues/podMonitor.enabled
+    value: true
+  - op: replace
+    path: /deploy/helm/releases/0/setValues/serviceMonitor.enabled
+    value: true
+  - op: replace
+    path: /deploy/helm/releases/0/setValues/grafana.dashboards.enabled
+    value: true
 - name: kind-opentelemetry
   patches:
   - op: replace