Backup prometheus data from CI run (#2636)

* Backup Prometheus data from CI run
* use `kube-prometheus-stack` instead of `prometheus`
* Add dump-analyzer script
* Fix monitor helm chart values
* Update readme for dump-analyzer
* Disable grafana and alertmanager

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

hack/backup-prometheus.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ -n "${TRACE-}" ]]; then
+    set -o xtrace
+fi
+NAMESPACE=${1:-monitoring}
+RELEASE=${2:-prometheus}
+BACKUP_DIR=${3:-/tmp/prometheus}
+LABELS="-l app=kube-prometheus-stack-${RELEASE}"
+
+sts_name=$(kubectl -n "$NAMESPACE" get sts "$LABELS" -o custom-columns=:.metadata.name | grep ^"$RELEASE")
+echo "Statefulset name: $sts_name"
+
+mkdir -p "$BACKUP_DIR"/prometheus
+# Copy snapshot to local
+kubectl -n "$NAMESPACE" cp "$sts_name"-0:/prometheus/ "$BACKUP_DIR"/prometheus
+
+mkdir -p "$BACKUP_DIR"/etc/prometheus
+kubectl -n "$NAMESPACE" cp "$sts_name"-0:/etc/prometheus "$BACKUP_DIR"/etc/prometheus