Removed call to InfluxDB via controller proxy (#2638)

* handle error condition in fission fn log command
* use the single stream for log exclude fetcher logs
* add all pods in the fn logs command
* update the previous stable version
* remove proxy to the controller for influxdb call
* stop running controller if influxdb flag turned true
* port-forward for influx DB
* remove controller dependency from test case
* port forward to influx DB if URL not provided by the user
* archive pruner test

charts/fission-all/templates/controller/clusterrole.yaml

@@ -1,4 +1,4 @@
-{{- if or (.Values.controller.enabled) (.Values.influxdb.enabled) }}
+{{- if .Values.controller.enabled }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:

charts/fission-all/templates/controller/clusterrolebinding.yaml

@@ -1,4 +1,4 @@
-{{- if or (.Values.controller.enabled) (.Values.influxdb.enabled) }}
+{{- if .Values.controller.enabled }}
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:

charts/fission-all/templates/controller/deployment.yaml

@@ -1,4 +1,4 @@
-{{- if or (.Values.controller.enabled) (.Values.influxdb.enabled) }}
+{{- if .Values.controller.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:

charts/fission-all/templates/controller/role-fission-cr.yaml

@@ -1,4 +1,4 @@
-{{- if or (.Values.controller.enabled) (.Values.influxdb.enabled) }}
+{{- if .Values.controller.enabled }}
 {{- include "fission-role-generator" (merge (dict "namespace" .Values.defaultNamespace "component" "controller") .) }}
 
 {{- if not .Values.singleDefaultNamespace }}

charts/fission-all/templates/controller/serviceaccount.yaml

@@ -1,4 +1,4 @@
-{{- if or (.Values.controller.enabled) (.Values.influxdb.enabled) }}
+{{- if .Values.controller.enabled }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:

charts/fission-all/templates/controller/servicemonitor.yaml

@@ -1,4 +1,4 @@
-{{- if or (.Values.controller.enabled) (.Values.influxdb.enabled) }}
+{{- if .Values.controller.enabled }}
 {{- if .Values.serviceMonitor.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor

charts/fission-all/templates/controller/svc.yaml

@@ -1,4 +1,4 @@
-{{- if or (.Values.controller.enabled) (.Values.influxdb.enabled) }}
+{{- if .Values.controller.enabled }}
 apiVersion: v1
 kind: Service
 metadata:

pkg/fission-cli/logdb/influxdb.go

@@ -22,8 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"net/url"
+	"os"
-	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -42,11 +41,20 @@ const (
 )
 
 func NewInfluxDB(ctx context.Context, logDBOptions LogDBOptions) (InfluxDB, error) {
-	server, err := util.GetApplicationUrl(ctx, logDBOptions.Client, "application=fission-api")
+
-	if err != nil {
+	dbType := INFLUXDB
-		return InfluxDB{}, err
+	// retrieve db auth config from the env
+	url := os.Getenv(fmt.Sprintf("%s_URL", dbType))
+	if url == "" {
+		// Portforward to the influxdb
+		localRouterPort, err := util.SetupPortForward(ctx, logDBOptions.Client, util.GetFissionNamespace(), "svc=influxdb")
+		if err != nil {
+			return InfluxDB{}, err
+		}
+		url = "http://127.0.0.1:" + localRouterPort + "/query"
 	}
-	return InfluxDB{endpoint: server}, nil
+
+	return InfluxDB{endpoint: url}, nil
 }
 
 type InfluxDB struct {
@@ -89,7 +97,7 @@ func (influx InfluxDB) GetLogs(ctx context.Context, filter LogFilter, output *by
 
 	query := influxdbClient.NewQueryWithParameters(queryCmd, INFLUXDB_DATABASE, "", parameters)
 	logEntries := []LogEntry{}
-	response, err := influx.query(query)
+	response, err := influx.query(ctx, query)
 	if err != nil {
 		return err
 	}
@@ -158,18 +166,16 @@ func (influx InfluxDB) GetLogs(ctx context.Context, filter LogFilter, output *by
 	return nil
 }
 
-func (influx InfluxDB) query(query influxdbClient.Query) (*influxdbClient.Response, error) {
+func (influx InfluxDB) query(ctx context.Context, query influxdbClient.Query) (*influxdbClient.Response, error) {
-	queryURL, err := url.Parse(influx.endpoint)
+
-	if err != nil {
+	username := os.Getenv(fmt.Sprintf("%s_USERNAME", INFLUXDB))
-		return nil, err
+	password := os.Getenv(fmt.Sprintf("%s_PASSWORD", INFLUXDB))
-	}
+
-	// connect to controller first, then controller will redirect our query command
+	req, err := http.NewRequest(http.MethodPost, influx.endpoint, nil)
-	// to influxdb and proxy back the db response.
-	queryURL.Path = path.Clean(fmt.Sprintf("%s/proxy/%s", queryURL.Path, INFLUXDB))
-	req, err := http.NewRequest(http.MethodPost, queryURL.String(), nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "error creating request for log proxy")
 	}
+	req.SetBasicAuth(username, password)
 
 	parametersBytes, err := json.Marshal(query.Parameters)
 	if err != nil {

skaffold.yaml

@@ -117,9 +117,6 @@ profiles:
   - op: replace
     path: /deploy/helm/releases/0/setValues/routerServiceType
     value: NodePort
-  - op: replace
-    path: /deploy/helm/releases/0/setValues/influxdb.enabled
-    value: true
   - op: replace
     path: /deploy/helm/releases/0/setValues/canaryDeployment.enabled
     value: true

test/tests/test_archive_pruner.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euo pipefail
+# set -euo pipefail
 source $(dirname $0)/../utils.sh
 
 TEST_ID=$(generate_test_id)
@@ -51,13 +51,12 @@ get_archive_url_from_package() {
     url=`kubectl -n default get package $1 -ojsonpath='{.spec.deployment.url}'`
 }
 
+urldecode() { : "${*//+/ }"; echo -e "${_//%/\\x}"; }
+
 get_archive_from_storage() {
-    storage_service_url=$1
+    # storage_service_url=$1
-    controller_ip=$CONTROLLER_IP
+    archive_url=$( urldecode $1)
-    controller_proxy_url=`echo $storage_service_url | sed -e "s/storagesvc.$FISSION_NAMESPACE/$controller_ip\/proxy\/storage/"`
+    fission archive list | grep $(echo "$archive_url" |cut -d= -f 2)| wc -l
-    log "controller_proxy_url=$controller_proxy_url"
-    http_status=`curl --retry 5 -sw "%{http_code}" $controller_proxy_url -o /dev/null`
-    echo "http_status: $http_status"
 }
 
 #1. declare trap to cleanup for EXIT
@@ -94,18 +93,18 @@ main() {
     log "deleted packages : $pkg_1 $pkg_2"
 
     # curl on the archive url
-    get_archive_from_storage $url_1
+    archiveCount=$(get_archive_from_storage $url_1)
-    log "http_status for $url_1 : $http_status"
+    log "recieved archive status for $url_1"
-    if [ "$http_status" -ne "200" ]; then
+    if [[ $archiveCount -eq 0 ]]; then
-        log "Archive $url_1 absent on storage, while expected to be present"
+        log "archive not found"
         exit 1
     fi
 
     # curl on the archive url
-    get_archive_from_storage $url_2
+     archiveCount=$(get_archive_from_storage $url_2)
-    log "http_status for $url_2 : $http_status"
+     log "recieved archive status for $url_2 "
-    if [ "$http_status" -ne "200" ]; then
+    if [[ $archiveCount -eq 0 ]]; then
-        log "Archive $url_2 absent on storage, while expected to be present"
+        log "archive not found"
         exit 1
     fi
 
@@ -114,20 +113,20 @@ main() {
     sleep 300
 
     # curl on the archive url
-    get_archive_from_storage $url_1
+    archiveCount=$(get_archive_from_storage $url_1) 
-    log "http_status for $url_1 : $http_status"
+    if [[ $archiveCount -ne 0 ]]; then
-    if [ "$http_status" -ne "404" ]; then
+        log "archive found"
-        log "Archive $url_1 should have been recycled, but curl returned $http_status, while expected status is 404."
         exit 1
     fi
+    log "archive pruned for $url_1 "
 
     # curl on the archive url
-    get_archive_from_storage $url_2
+    archiveCount=$(get_archive_from_storage $url_2)
-    log "http_status for $url_2 : $http_status"
+    if [[ $archiveCount -ne 0 ]]; then
-    if [ "$http_status" -ne "404" ]; then
+        log "archive found"
-        log "Archive $url_2 should have been recycled, but curl returned $http_status, while expected status is 404."
         exit 1
     fi
+    log "archive pruned for $url_2 "
 
     log "Test archive pruner PASSED"
 }