Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backup prometheus data from CI run (#2636)

Browse Source 
* Backup Prometheus data from CI run
* use `kube-prometheus-stack` instead of `prometheus`
* Add dump-analyzer script
* Fix monitor helm chart values
* Update readme for dump-analyzer
* Disable grafana and alertmanager

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Sanket Sudake
2022-11-25 15:48:09 +05:30
committed by GitHub
parent 8db3d0065a
commit f11902e81b
7 changed files with 393 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/push_pr.yaml
+17 -1
View File
@@ -91,7 +91,8 @@ jobs:
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
kubectl create ns monitoring
helm install monitoring prometheus-community/prometheus -n monitoring
helm install prometheus prometheus-community/kube-prometheus-stack -n monitoring \
--set grafana.enabled=false --set alertmanager.enabled=false
- name: Build and Install Fission CLI
run: |
@@ -133,6 +134,12 @@ jobs:
run: |
kind export logs --name kind kind-logs
- name: Backup prometheus data
timeout-minutes: 10
if: ${{ always() }}
run: |
TRACE=1 ./hack/backup-prometheus.sh
- name: Archive fission dump
timeout-minutes: 10
if: ${{ failure() || cancelled() }}
@@ -142,6 +149,15 @@ jobs:
path: fission-dump/*.zip
retention-days: 5
- name: Archive prometheus dump
timeout-minutes: 10
if: ${{ always() }}
uses: actions/upload-artifact@v3
with:
name: prom-dump-${{ github.run_id }}-${{ matrix.kindversion }}
path: /tmp/prometheus/*
retention-days: 5
- name: Archive kind logs
timeout-minutes: 10
if: ${{ always() }}
.gitignore
+3
View File
@@ -37,3 +37,6 @@ manifest/
coverage.txt
cosign.key
# Dumps
.dumps/
cmd/fission-bundle/main.go
+1 -1
View File
@@ -216,7 +216,7 @@ Options:
ctx := signals.SetupSignalHandler()
profile.ProfileIfEnabled(ctx, logger)
version := fmt.Sprintf("Fission Bundle Version: %v", info.BuildInfo().String())
version := fmt.Sprintf("Fission Bundle Version: %s", info.BuildInfo().String())
arguments, err := docopt.ParseArgs(usage, nil, version)
if err != nil {
logger.Error("failed to parse arguments", zap.Error(err))
hack/backup-prometheus.sh
Executable
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
if [[ -n "${TRACE-}" ]]; then
set -o xtrace
fi
NAMESPACE=${1:-monitoring}
RELEASE=${2:-prometheus}
BACKUP_DIR=${3:-/tmp/prometheus}
LABELS="-l app=kube-prometheus-stack-${RELEASE}"
sts_name=$(kubectl -n "$NAMESPACE" get sts "$LABELS" -o custom-columns=:.metadata.name | grep ^"$RELEASE")
echo "Statefulset name: $sts_name"
mkdir -p "$BACKUP_DIR"/prometheus
# Copy snapshot to local
kubectl -n "$NAMESPACE" cp "$sts_name"-0:/prometheus/ "$BACKUP_DIR"/prometheus
mkdir -p "$BACKUP_DIR"/etc/prometheus
kubectl -n "$NAMESPACE" cp "$sts_name"-0:/etc/prometheus "$BACKUP_DIR"/etc/prometheus
hack/dump-analyzer
Executable
+199
View File
@@ -0,0 +1,199 @@
#!/usr/bin/env bash
###
# This script is used to analyze the dump files generated by the Fission CI.
###
set -o errexit
set -o nounset
set -o pipefail
if [[ -n "${TRACE-}" ]]; then
set -o xtrace
fi
REPO="fission/fission"
function exit_error() {
echo "$1"
exit 1
}
function doit() {
echo "! $*"
"$@"
}
function check_context() {
if [ -z "$DUMP_CONTEXT" ]; then
exit_error "DUMP_CONTEXT not set"
fi
}
function prom_context() {
if [ -z "$PROM_CONTEXT" ]; then
exit_error "PROM_CONTEXT not set"
fi
}
function list_context() {
for dump in $(find .dumps/$RUN_ID/extract -type d -depth 1); do
echo "==== $dump ===="
echo "-- Fission Version --"
cat $dump/fission-version/fission-version.txt | grep "server:\|client\:\|Version"
echo "-- K8s Version --"
cat $dump/kubernetes-version/kubernetes-version.txt | grep gitVersion
done
}
function list_kind() {
for dump in $(find .dumps/$RUN_ID -type d -depth 1 -name 'kind-logs-*'); do
echo "==== $dump ===="
cat $dump/kind-version.txt
echo -e "\n"
done
}
function extract() {
ARTIFACT_PATH=.dumps/$RUN_ID
if [ ! -d "$ARTIFACT_PATH" ]; then
doit mkdir -p "$ARTIFACT_PATH"
fi
echo $ARTIFACT_PATH
# Don't download if artifact already exits for run
if [ ! -d "$ARTIFACT_PATH"/fission-dump/ ]; then
doit gh run download "$RUN_ID" -R "$REPO" -D "$ARTIFACT_PATH"
fi
if [ -d "$ARTIFACT_PATH"/extract ]; then
doit rm -r "$ARTIFACT_PATH"/extract
fi
DUMPS=$(find "$ARTIFACT_PATH" -name 'fission-dump*.zip')
for dump in $DUMPS; do
doit unzip -q $dump -d "$ARTIFACT_PATH"/extract
done
}
function info() {
check_context
echo "Dump context: $DUMP_CONTEXT"
doit cat "$DUMP_CONTEXT"/fission-version/fission-version.txt
doit cat "$DUMP_CONTEXT"/kubernetes-version/kubernetes-version.txt
}
function race_conditions() {
check_context
# doit ack -A 10 "WARNING: DATA RACE" "$DUMP_CONTEXT/"
LOG_FILES=$(find $DUMP_CONTEXT -name '*.txt' -type f | grep log)
for logfile in $LOG_FILES; do
out=$(sed -n '/DATA RACE/,/==================$/p' "$logfile")
if [ "$out" != "" ]; then
shortname=$(basename $logfile)
echo "Trace from $shortname"
sed -n '/DATA RACE/,/==================$/p' "$logfile"
fi
done
}
function error_logs() {
check_context
# ack '"level":"error"' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | go-slearch -pts,caller,msg,error,logger,errorVerbose | sort
ack -i 'error' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | sort
}
function prometheus_run() {
# check if prometheus binary exists
if command -v prometheus &> /dev/null ; then
echo "prometheus binary found"
else
exit_error "prometheus binary not found"
fi
prom_context
TSDB_PATH=$PROM_CONTEXT/prometheus
ETC_PROM=$PROM_CONTEXT/etc/prometheus
prometheus \
--web.console.templates=$ETC_PROM/consoles \
--web.console.libraries=$ETC_PROM/console_libraries \
--storage.tsdb.retention.time=10d \
--config.file=$ETC_PROM/config_out/prometheus.env.yaml \
--storage.tsdb.path=${TSDB_PATH}/ \
--web.enable-lifecycle \
--web.listen-address=127.0.0.1:9090 \
--web.route-prefix=/ \
--storage.tsdb.wal-compression \
--web.config.file=$ETC_PROM/web_config/web-config.yaml
}
function usage {
echo "./$(basename "$0") [OPTIONS]"
echo "Utilities related to fission dump analysis"
echo "
Options:
-h Show usage
-x [run_id] Download and extract dump locally
-l [run_id] List dump files for a run
-k [run_id] List kind exports in a run
Following options required DUMP_CONTEXT variable set.
-i Display dump info
-r Find all race conditions in dump
-e Find all errors in logs
Following options require PROM_CONTEXT to be set.
-p Run prometheus with the dump"
exit 3
}
# list of arguments expected in the input
optstring=":hxirlekp"
if [[ ${#} -eq 0 ]]; then
usage
fi
while getopts ${optstring} arg; do
case ${arg} in
h)
echo "showing usage!"
usage
;;
x)
RUN_ID=$2
if [ -z "$RUN_ID" ]; then
exit_error "run id not mentioned"
fi
extract
;;
l)
RUN_ID=$2
if [ -z "$RUN_ID" ]; then
exit_error "run id not mentioned"
fi
list_context
;;
i)
info
;;
r)
race_conditions
;;
e)
error_logs
;;
k)
RUN_ID=$2
if [ -z "$RUN_ID" ]; then
exit_error "run id not mentioned"
fi
list_kind
;;
p)
prometheus_run
;;
:)
echo "$0: Must supply an argument to -$OPTARG." >&2
exit 1
;;
?)
echo "Invalid option: -${OPTARG}."
exit 2
;;
esac
done
hack/dump-analyzer.md
+133
View File
@@ -0,0 +1,133 @@
# Dump analyzer
## Usage
```text
./hack/dump-analyzer -h
showing usage!
./dump-analyzer [OPTIONS]
Utilities related to fission dump analysis
Options:
-h Show usage
-x [run_id] Download and extract dump locally
-l [run_id] List dump files for a run
-k [run_id] List kind exports in a run
Following options required DUMP_CONTEXT variable set.
-i Display dump info
-r Find all race conditions in dump
-e Find all errors in logs
Following options require PROM_CONTEXT to be set.
-p Run prometheus with the dump
```
## Downloading dump
```sh
$ ./hack/dump-analyzer -x 3179903728
! mkdir -p .dumps/3179903728
.dumps/3179903728
! gh run download 3179903728 -R fission/fission -D .dumps/3179903728
! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.19.16/fission-dump_1664866624.zip -d .dumps/3179903728/extract
! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.21.12/fission-dump_1664866628.zip -d .dumps/3179903728/extract
! unzip -q .dumps/3179903728/fission-dump-3179903728-v1.20.15/fission-dump_1664866662.zip -d .dumps/3179903728/extract
```
## Check all dumps
```sh
./hack/dump-analyzer -l 3179903728
==== .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304 ====
-- Fission Version --
client:
Version: v0.0.0
server:
Version: v0.0.0
-- K8s Version --
gitVersion: v1.21.12
==== .dumps/3179903728/extract/e7febf06-79ce-4eb7-bba0-3fc66aa4bf542096519315 ====
-- Fission Version --
client:
Version: v0.0.0
server:
Version: v0.0.0
-- K8s Version --
gitVersion: v1.19.16
==== .dumps/3179903728/extract/3b339dc2-e9dd-4f7a-a721-41e16d9d248f4152710428 ====
-- Fission Version --
client:
Version: v0.0.0
server:
Version: v0.0.0
-- K8s Version --
gitVersion: v1.20.15
```
We can see that the dump is from 3 different versions of k8s.
To explore specific dump set `DUMP_CONTEXT` environment variable.
```sh
$ export DUMP_CONTEXT=.dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304
$ ./hack/dump-analyzer -i
Dump context: .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304
! cat .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304/fission-version/fission-version.txt
client:
fission/core:
BuildDate: "2022-10-04T06:28:54Z"
GitCommit: 9e74b01
Version: v0.0.0
server:
fission/core:
BuildDate: "2022-10-04T06:29:01Z"
GitCommit: 9e74b01
Version: v0.0.0
! cat .dumps/3179903728/extract/4b24c3db-b5d8-43a7-858d-48746530d29e2534094304/kubernetes-version/kubernetes-version.txt
buildDate: "2022-05-19T20:02:29Z"
compiler: gc
gitCommit: 696a9fdd2a58340e61e0d815c5769d266fca0802
gitTreeState: clean
gitVersion: v1.21.12
goVersion: go1.16.15
major: "1"
minor: "21"
platform: linux/amd64
# See all errors in the dump
$ ./hack/dump-analyzer -e
# If you want to see error in specific dump you can also grep
$ grep -rin "string" $DUMP_CONTEXT
# OR
$ ack "string" $DUMP_CONTEXT
```
## Kind logs
```sh
./hack/dump-analyzer -k 3179903728
==== .dumps/3179903728/kind-logs-3179903728-v1.20.15 ====
kind v0.14.0 go1.18.2 linux/amd64
==== .dumps/3179903728/kind-logs-3179903728-v1.19.16 ====
kind v0.14.0 go1.18.2 linux/amd64
==== .dumps/3179903728/kind-logs-3179903728-v1.21.12 ====
kind v0.14.0 go1.18.2 linux/amd64
```
## Prometheus with dump
```sh
$ export DUMP_CONTEXT=.dumps/3539792658/prom-dump-3539792658-v1.21.14
$ ./hack/dump-analyzer -p
# Visit localhost:9090
```
skaffold.yaml
+17 -1
View File
@@ -66,6 +66,13 @@ deploy:
webhook.securityContext.enabled: true
storagesvc.securityContext.enabled: true
controller.enabled: false
serviceMonitor.enabled: false
serviceMonitor.namespace: monitoring
serviceMonitor.additionalServiceMonitorLabels.release: prometheus
podMonitor.enabled: false
podMonitor.namespace: monitoring
podMonitor.additionalPodMonitorLabels.release: prometheus
grafana.dashboards.enabled: false
wait: true
flags:
install:
@@ -118,7 +125,16 @@ profiles:
value: true
- op: replace
path: /deploy/helm/releases/0/setValues/prometheus.serviceEndpoint
value: "http://monitoring-prometheus-server.monitoring.svc.cluster.local"
value: "http://prometheus-operated.monitoring.svc.cluster.local:9090"
- op: replace
path: /deploy/helm/releases/0/setValues/podMonitor.enabled
value: true
- op: replace
path: /deploy/helm/releases/0/setValues/serviceMonitor.enabled
value: true
- op: replace
path: /deploy/helm/releases/0/setValues/grafana.dashboards.enabled
value: true
- name: kind-opentelemetry
patches:
- op: replace