fission-src/hack/dump-analyzer

#!/usr/bin/env bash
###
# This script is used to analyze the dump files generated by the Fission CI.
###
set -o errexit
set -o nounset
set -o pipefail
if [[ -n "${TRACE-}" ]]; then
    set -o xtrace
fi
REPO="fission/fission"

function exit_error() {
    echo "$1"
    exit 1
}

function doit() {
    echo "! $*"
    "$@"
}

function check_context() {
    if [ -z "$DUMP_CONTEXT" ]; then
        exit_error "DUMP_CONTEXT not set"
    fi
}

function prom_context() {
    if [ -z "$PROM_CONTEXT" ]; then
        exit_error "PROM_CONTEXT not set"
    fi
    echo "Prometheus context: $PROM_CONTEXT"
}

function list_context() {
    for dump in $(find .dumps/$RUN_ID/extract -type d -depth 1); do
        echo "==== $dump ===="
        echo "-- Fission Version --"
        cat $dump/fission-version/fission-version.txt | grep "server:\|client\:\|Version"
        echo "-- K8s Version --"
        cat $dump/kubernetes-version/kubernetes-version.txt | grep gitVersion
    done
}

function list_kind() {
    for dump in $(find .dumps/$RUN_ID -type d -depth 1 -name 'kind-logs-*'); do
        echo "==== $dump ===="
        cat $dump/kind-version.txt
        echo -e "\n"
    done
}

function extract() {
    ARTIFACT_PATH=.dumps/$RUN_ID
    if [ ! -d "$ARTIFACT_PATH" ]; then
        doit mkdir -p "$ARTIFACT_PATH"
    fi
    echo $ARTIFACT_PATH
    # Don't download if artifact already exits for run
    if [ ! -d "$ARTIFACT_PATH"/fission-dump/ ]; then
        doit gh run download "$RUN_ID" -R "$REPO" -D "$ARTIFACT_PATH"
    fi
    if [ -d "$ARTIFACT_PATH"/extract ]; then
        doit rm -r "$ARTIFACT_PATH"/extract
    fi
    DUMPS=$(find "$ARTIFACT_PATH" -name 'fission-dump*.zip')
    for dump in $DUMPS; do
        doit unzip -q $dump -d "$ARTIFACT_PATH"/extract
    done
}

function info() {
    check_context
    echo "Dump context: $DUMP_CONTEXT"
    doit cat "$DUMP_CONTEXT"/fission-version/fission-version.txt
    doit cat "$DUMP_CONTEXT"/kubernetes-version/kubernetes-version.txt
}

function race_conditions() {
    check_context
    # doit ack -A 10 "WARNING: DATA RACE" "$DUMP_CONTEXT/"
    LOG_FILES=$(find $DUMP_CONTEXT -name '*.txt' -type f | grep log)
    for logfile in $LOG_FILES; do
        out=$(sed -n '/DATA RACE/,/==================$/p' "$logfile")
        if [ "$out" != "" ]; then
            shortname=$(basename $logfile)
            echo "Trace from $shortname"
            sed -n '/DATA RACE/,/==================$/p' "$logfile"
        fi
    done
}

function error_logs() {
    check_context
    # ack '"level":"error"' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | go-slearch -pts,caller,msg,error,logger,errorVerbose | sort
    ack -i 'error' $DUMP_CONTEXT | grep -v "404" | cut -d':' -f 3- | sort
}

function prometheus_run() {
    # check if prometheus binary exists
    if command -v prometheus &> /dev/null ; then
        echo "prometheus binary found"
    else
        exit_error "prometheus binary not found"
    fi
    prom_context
    TSDB_PATH=$PROM_CONTEXT/prometheus
    ETC_PROM=$PROM_CONTEXT/etc/prometheus
    prometheus \
        --web.console.templates=$ETC_PROM/consoles \
        --web.console.libraries=$ETC_PROM/console_libraries \
        --storage.tsdb.retention.time=10d \
        --config.file=$ETC_PROM/config_out/prometheus.env.yaml \
        --storage.tsdb.path=${TSDB_PATH}/ \
        --web.enable-lifecycle \
        --web.listen-address=127.0.0.1:${PROM_PORT:-9090} \
        --web.route-prefix=/ \
        --storage.tsdb.wal-compression \
        --web.config.file=$ETC_PROM/web_config/web-config.yaml
}

function usage {
    echo "./$(basename "$0") [OPTIONS]"
    echo "Utilities related to fission dump analysis"
    echo "
Options:
    -h              Show usage
    -x [run_id]     Download and extract dump locally
    -l [run_id]    List dump files for a run
    -k [run_id]    List kind exports in a run

    Following options required DUMP_CONTEXT variable set.

    -i              Display dump info
    -r              Find all race conditions in dump
    -e              Find all errors in logs

    Following options require PROM_CONTEXT to be set.

    -p             Run prometheus with the dump"
    exit 3
}

# list of arguments expected in the input
optstring=":hxirlekp"

if [[ ${#} -eq 0 ]]; then
    usage
fi

while getopts ${optstring} arg; do
    case ${arg} in
    h)
        echo "showing usage!"
        usage
        ;;
    x)
        RUN_ID=$2
        if [ -z "$RUN_ID" ]; then
            exit_error "run id not mentioned"
        fi
        extract
        ;;
    l)
        RUN_ID=$2
        if [ -z "$RUN_ID" ]; then
            exit_error "run id not mentioned"
        fi
        list_context
        ;;
    i)
        info
        ;;
    r)
        race_conditions
        ;;
    e)
        error_logs
        ;;
    k)
        RUN_ID=$2
        if [ -z "$RUN_ID" ]; then
            exit_error "run id not mentioned"
        fi
        list_kind
        ;;
    p)
        ARG_PROM_CONTEXT=$2
        if [ ! -z "$ARG_PROM_CONTEXT" ]; then
            PROM_CONTEXT=$ARG_PROM_CONTEXT
        fi
        prometheus_run
        ;;
    :)
        echo "$0: Must supply an argument to -$OPTARG." >&2
        exit 1
        ;;
    ?)
        echo "Invalid option: -${OPTARG}."
        exit 2
        ;;
    esac
done