Backup prometheus data from CI run (#2636)
* Backup Prometheus data from CI run * use `kube-prometheus-stack` instead of `prometheus` * Add dump-analyzer script * Fix monitor helm chart values * Update readme for dump-analyzer * Disable grafana and alertmanager Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Sanket Sudake
Executable
+199
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env bash
|
||||
###
|
||||
# This script is used to analyze the dump files generated by the Fission CI.
|
||||
###
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
set -o pipefail
|
||||
if [[ -n "${TRACE-}" ]]; then
|
||||
set -o xtrace
|
||||
fi
|
||||
REPO="fission/fission"
|
||||
|
||||
function exit_error() {
|
||||
echo "$1"
|
||||
exit 1
|
||||
}
|
||||
|
||||
function doit() {
|
||||
echo "! $*"
|
||||
"$@"
|
||||
}
|
||||
|
||||
function check_context() {
|
||||
if [ -z "$DUMP_CONTEXT" ]; then
|
||||
exit_error "DUMP_CONTEXT not set"
|
||||
fi
|
||||
}
|
||||
|
||||
function prom_context() {
|
||||
if [ -z "$PROM_CONTEXT" ]; then
|
||||
exit_error "PROM_CONTEXT not set"
|
||||
fi
|
||||
}
|
||||
|
||||
function list_context() {
|
||||
for dump in $(find .dumps/$RUN_ID/extract -type d -depth 1); do
|
||||
echo "==== $dump ===="
|
||||
echo "-- Fission Version --"
|
||||
cat $dump/fission-version/fission-version.txt | grep "server:\|client\:\|Version"
|
||||
echo "-- K8s Version --"
|
||||
cat $dump/kubernetes-version/kubernetes-version.txt | grep gitVersion
|
||||
done
|
||||
}
|
||||
|
||||
function list_kind() {
|
||||
for dump in $(find .dumps/$RUN_ID -type d -depth 1 -name 'kind-logs-*'); do
|
||||
echo "==== $dump ===="
|
||||
cat $dump/kind-version.txt
|
||||
echo -e "\n"
|
||||
done
|
||||
}
|
||||
|
||||
function extract() {
|
||||
ARTIFACT_PATH=.dumps/$RUN_ID
|
||||
if [ ! -d "$ARTIFACT_PATH" ]; then
|
||||
doit mkdir -p "$ARTIFACT_PATH"
|
||||
fi
|
||||
echo $ARTIFACT_PATH
|
||||
# Don't download if artifact already exits for run
|
||||
if [ ! -d "$ARTIFACT_PATH"/fission-dump/ ]; then
|
||||
doit gh run download "$RUN_ID" -R "$REPO" -D "$ARTIFACT_PATH"
|
||||
fi
|
||||
if [ -d "$ARTIFACT_PATH"/extract ]; then
|
||||
doit rm -r "$ARTIFACT_PATH"/extract
|
||||
fi
|
||||
DUMPS=$(find "$ARTIFACT_PATH" -name 'fission-dump*.zip')
|
||||
for dump in $DUMPS; do
|
||||
doit unzip -q $dump -d "$ARTIFACT_PATH"/extract
|
||||
done
|
||||
}
|
||||
|
||||
function info() {
|
||||
check_context
|
||||
echo "Dump context: $DUMP_CONTEXT"
|
||||
doit cat "$DUMP_CONTEXT"/fission-version/fission-version.txt
|
||||
doit cat "$DUMP_CONTEXT"/kubernetes-version/kubernetes-version.txt
|
||||
}
|
||||
|
||||
function race_conditions() {
|
||||
check_context
|
||||
# doit ack -A 10 "WARNING: DATA RACE" "$DUMP_CONTEXT/"
|
||||
LOG_FILES=$(find $DUMP_CONTEXT -name '*.txt' -type f | grep log)
|
||||
for logfile in $LOG_FILES; do
|
||||
out=$(sed -n '/DATA RACE/,/==================$/p' "$logfile")
|
||||
if [ "$out" != "" ]; then
|
||||
shortname=$(basename $logfile)
|
||||
echo "Trace from $shortname"
|
||||
sed -n '/DATA RACE/,/==================$/p' "$logfile"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
function error_logs() {
|
||||
check_context
|
||||
# ack '"level":"error"' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | go-slearch -pts,caller,msg,error,logger,errorVerbose | sort
|
||||
ack -i 'error' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | sort
|
||||
}
|
||||
|
||||
function prometheus_run() {
|
||||
# check if prometheus binary exists
|
||||
if command -v prometheus &> /dev/null ; then
|
||||
echo "prometheus binary found"
|
||||
else
|
||||
exit_error "prometheus binary not found"
|
||||
fi
|
||||
prom_context
|
||||
TSDB_PATH=$PROM_CONTEXT/prometheus
|
||||
ETC_PROM=$PROM_CONTEXT/etc/prometheus
|
||||
prometheus \
|
||||
--web.console.templates=$ETC_PROM/consoles \
|
||||
--web.console.libraries=$ETC_PROM/console_libraries \
|
||||
--storage.tsdb.retention.time=10d \
|
||||
--config.file=$ETC_PROM/config_out/prometheus.env.yaml \
|
||||
--storage.tsdb.path=${TSDB_PATH}/ \
|
||||
--web.enable-lifecycle \
|
||||
--web.listen-address=127.0.0.1:9090 \
|
||||
--web.route-prefix=/ \
|
||||
--storage.tsdb.wal-compression \
|
||||
--web.config.file=$ETC_PROM/web_config/web-config.yaml
|
||||
}
|
||||
|
||||
function usage {
|
||||
echo "./$(basename "$0") [OPTIONS]"
|
||||
echo "Utilities related to fission dump analysis"
|
||||
echo "
|
||||
Options:
|
||||
-h Show usage
|
||||
-x [run_id] Download and extract dump locally
|
||||
-l [run_id] List dump files for a run
|
||||
-k [run_id] List kind exports in a run
|
||||
|
||||
Following options required DUMP_CONTEXT variable set.
|
||||
|
||||
-i Display dump info
|
||||
-r Find all race conditions in dump
|
||||
-e Find all errors in logs
|
||||
|
||||
Following options require PROM_CONTEXT to be set.
|
||||
|
||||
-p Run prometheus with the dump"
|
||||
exit 3
|
||||
}
|
||||
|
||||
# list of arguments expected in the input
|
||||
optstring=":hxirlekp"
|
||||
|
||||
if [[ ${#} -eq 0 ]]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
while getopts ${optstring} arg; do
|
||||
case ${arg} in
|
||||
h)
|
||||
echo "showing usage!"
|
||||
usage
|
||||
;;
|
||||
x)
|
||||
RUN_ID=$2
|
||||
if [ -z "$RUN_ID" ]; then
|
||||
exit_error "run id not mentioned"
|
||||
fi
|
||||
extract
|
||||
;;
|
||||
l)
|
||||
RUN_ID=$2
|
||||
if [ -z "$RUN_ID" ]; then
|
||||
exit_error "run id not mentioned"
|
||||
fi
|
||||
list_context
|
||||
;;
|
||||
i)
|
||||
info
|
||||
;;
|
||||
r)
|
||||
race_conditions
|
||||
;;
|
||||
e)
|
||||
error_logs
|
||||
;;
|
||||
k)
|
||||
RUN_ID=$2
|
||||
if [ -z "$RUN_ID" ]; then
|
||||
exit_error "run id not mentioned"
|
||||
fi
|
||||
list_kind
|
||||
;;
|
||||
p)
|
||||
prometheus_run
|
||||
;;
|
||||
:)
|
||||
echo "$0: Must supply an argument to -$OPTARG." >&2
|
||||
exit 1
|
||||
;;
|
||||
?)
|
||||
echo "Invalid option: -${OPTARG}."
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
Reference in New Issue
Block a user