#!/usr/bin/env bash ### # This script is used to analyze the dump files generated by the Fission CI. ### set -o errexit set -o nounset set -o pipefail if [[ -n "${TRACE-}" ]]; then set -o xtrace fi REPO="fission/fission" function exit_error() { echo "$1" exit 1 } function doit() { echo "! $*" "$@" } function check_context() { if [ -z "$DUMP_CONTEXT" ]; then exit_error "DUMP_CONTEXT not set" fi } function prom_context() { if [ -z "$PROM_CONTEXT" ]; then exit_error "PROM_CONTEXT not set" fi echo "Prometheus context: $PROM_CONTEXT" } function list_context() { for dump in $(find .dumps/$RUN_ID/extract -type d -depth 1); do echo "==== $dump ====" echo "-- Fission Version --" cat $dump/fission-version/fission-version.txt | grep "server:\|client\:\|Version" echo "-- K8s Version --" cat $dump/kubernetes-version/kubernetes-version.txt | grep gitVersion done } function list_kind() { for dump in $(find .dumps/$RUN_ID -type d -depth 1 -name 'kind-logs-*'); do echo "==== $dump ====" cat $dump/kind-version.txt echo -e "\n" done } function extract() { ARTIFACT_PATH=.dumps/$RUN_ID if [ ! -d "$ARTIFACT_PATH" ]; then doit mkdir -p "$ARTIFACT_PATH" fi echo $ARTIFACT_PATH # Don't download if artifact already exits for run if [ ! -d "$ARTIFACT_PATH"/fission-dump/ ]; then doit gh run download "$RUN_ID" -R "$REPO" -D "$ARTIFACT_PATH" fi if [ -d "$ARTIFACT_PATH"/extract ]; then doit rm -r "$ARTIFACT_PATH"/extract fi DUMPS=$(find "$ARTIFACT_PATH" -name 'fission-dump*.zip') for dump in $DUMPS; do doit unzip -q $dump -d "$ARTIFACT_PATH"/extract done } function info() { check_context echo "Dump context: $DUMP_CONTEXT" doit cat "$DUMP_CONTEXT"/fission-version/fission-version.txt doit cat "$DUMP_CONTEXT"/kubernetes-version/kubernetes-version.txt } function race_conditions() { check_context # doit ack -A 10 "WARNING: DATA RACE" "$DUMP_CONTEXT/" LOG_FILES=$(find $DUMP_CONTEXT -name '*.txt' -type f | grep log) for logfile in $LOG_FILES; do out=$(sed -n '/DATA RACE/,/==================$/p' "$logfile") if [ "$out" != "" ]; then shortname=$(basename $logfile) echo "Trace from $shortname" sed -n '/DATA RACE/,/==================$/p' "$logfile" fi done } function error_logs() { check_context # ack '"level":"error"' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | go-slearch -pts,caller,msg,error,logger,errorVerbose | sort ack -i 'error' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | sort } function prometheus_run() { # check if prometheus binary exists if command -v prometheus &> /dev/null ; then echo "prometheus binary found" else exit_error "prometheus binary not found" fi prom_context TSDB_PATH=$PROM_CONTEXT/prometheus ETC_PROM=$PROM_CONTEXT/etc/prometheus prometheus \ --web.console.templates=$ETC_PROM/consoles \ --web.console.libraries=$ETC_PROM/console_libraries \ --storage.tsdb.retention.time=10d \ --config.file=$ETC_PROM/config_out/prometheus.env.yaml \ --storage.tsdb.path=${TSDB_PATH}/ \ --web.enable-lifecycle \ --web.listen-address=127.0.0.1:${PROM_PORT:-9090} \ --web.route-prefix=/ \ --storage.tsdb.wal-compression \ --web.config.file=$ETC_PROM/web_config/web-config.yaml } function usage { echo "./$(basename "$0") [OPTIONS]" echo "Utilities related to fission dump analysis" echo " Options: -h Show usage -x [run_id] Download and extract dump locally -l [run_id] List dump files for a run -k [run_id] List kind exports in a run Following options required DUMP_CONTEXT variable set. -i Display dump info -r Find all race conditions in dump -e Find all errors in logs Following options require PROM_CONTEXT to be set. -p Run prometheus with the dump" exit 3 } # list of arguments expected in the input optstring=":hxirlekp" if [[ ${#} -eq 0 ]]; then usage fi while getopts ${optstring} arg; do case ${arg} in h) echo "showing usage!" usage ;; x) RUN_ID=$2 if [ -z "$RUN_ID" ]; then exit_error "run id not mentioned" fi extract ;; l) RUN_ID=$2 if [ -z "$RUN_ID" ]; then exit_error "run id not mentioned" fi list_context ;; i) info ;; r) race_conditions ;; e) error_logs ;; k) RUN_ID=$2 if [ -z "$RUN_ID" ]; then exit_error "run id not mentioned" fi list_kind ;; p) ARG_PROM_CONTEXT=$2 if [ ! -z "$ARG_PROM_CONTEXT" ]; then PROM_CONTEXT=$ARG_PROM_CONTEXT fi prometheus_run ;; :) echo "$0: Must supply an argument to -$OPTARG." >&2 exit 1 ;; ?) echo "Invalid option: -${OPTARG}." exit 2 ;; esac done