Return the error on failed specializations with fn test --debug (#917)

charts/fission-all/templates/deployment.yaml

@@ -104,7 +104,6 @@ metadata:
   namespace: {{ .Values.builderNamespace }}
 
 ---
-# TODO : Configure controller with prometheus endpoint
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -185,6 +184,8 @@ spec:
             value: {{ .Values.routerRoundTripKeepAliveTime | default "30s" | quote }}
           - name: ROUTER_ROUND_TRIP_MAX_RETRIES
             value: {{ .Values.routerRoundTripMaxRetries | default 10 | quote }}
+          - name: DEBUG_ENV
+            value: {{ .Values.debugEnv | quote }}
         readinessProbe:
           httpGet:
             path: "/router-healthz"

charts/fission-all/values.yaml

@@ -99,3 +99,7 @@ pruneInterval: 60
 
 ## Fission pre-install/pre-upgrade checks live in this image
 preUpgradeChecksImage: fission/pre-upgrade-checks
+
+## if there are any pod specialization errors when a function is triggered and this flag is set to true, the error
+## summary is returned as part of http response
+debugEnv: true

charts/fission-core/templates/deployment.yaml

@@ -180,6 +180,8 @@ spec:
             value: {{ .Values.routerRoundTripKeepAliveTime | default "30s" | quote }}
           - name: ROUTER_ROUND_TRIP_MAX_RETRIES
             value: {{ .Values.routerRoundTripMaxRetries | default 10 | quote }}
+          - name: DEBUG_ENV
+            value: {{ .Values.debugEnv | quote }}
         readinessProbe:
           httpGet:
             path: "/router-healthz"

charts/fission-core/values.yaml

@@ -65,3 +65,7 @@ pruneInterval: 60
 
 ## Fission pre-install/pre-upgrade checks live in this image
 preUpgradeChecksImage: fission/pre-upgrade-checks
+
+## if there are any pod specialization errors when a function is triggered and this flag is set to true, the error
+## summary is returned as part of http response
+debugEnv: true

executor/client/client.go

@@ -67,6 +67,7 @@ func (c *Client) GetServiceForFunction(metadata *metav1.ObjectMeta) (string, err
 
 	svcName, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
+		log.Printf("Returning from ioutil read body")
 		return "", err
 	}
 

fission/function.go

@@ -756,7 +756,9 @@ func fnTest(c *cli.Context) error {
 		functionUrl.RawQuery = query.Encode()
 	}
 
-	resp := httpRequest(c.String("method"), functionUrl.String(), c.String("body"), c.StringSlice("header"))
+	headers := c.StringSlice("header")
+
+	resp := httpRequest(c.String("method"), functionUrl.String(), c.String("body"), headers)
 	if resp.StatusCode < 400 {
 		body, err := ioutil.ReadAll(resp.Body)
 		util.CheckErr(err, "Function test")

router/functionHandler.go

@@ -63,6 +63,7 @@ type functionHandler struct {
 	fnWeightDistributionList []FunctionWeightDistribution
 	tsRoundTripperParams     *tsRoundTripperParams
 	recorderName             string
+	isDebugEnv               bool
 }
 
 // A layer on top of http.DefaultTransport, with retries.
@@ -165,10 +166,26 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
 			// send a request to executor to specialize a new pod
 			service, err := roundTripper.funcHandler.executor.GetServiceForFunction(
 				roundTripper.funcHandler.function)
+
 			if err != nil {
-				log.Printf("Err from GetServiceForFunction : %v", err)
+				statusCode, errMsg := fission.GetHTTPError(err)
+				log.Printf("Err from GetServiceForFunction : %v : %v", statusCode, errMsg)
+
 				// We might want a specific error code or header for fission failures as opposed to
 				// user function bugs.
+				if roundTripper.funcHandler.isDebugEnv {
+					return &http.Response{
+						StatusCode:    statusCode,
+						Proto:         req.Proto,
+						ProtoMajor:    req.ProtoMajor,
+						ProtoMinor:    req.ProtoMinor,
+						Body:          ioutil.NopCloser(bytes.NewBufferString(errMsg)),
+						ContentLength: int64(len(errMsg)),
+						Request:       req,
+						Header:        make(http.Header, 0),
+					}, nil
+				}
+
 				return nil, err
 			}
 
@@ -179,7 +196,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
 			}
 
 			// add the address in router's cache
-			log.Printf("assigning serviceUrl : %s for function : %s", service, roundTripper.funcHandler.function.Name)
+			log.Printf("assigning serviceUrl : %s for function : %s", serviceUrl, roundTripper.funcHandler.function.Name)
 			roundTripper.funcHandler.fmap.assign(roundTripper.funcHandler.function, serviceUrl)
 
 			// flag denotes that service was not obtained from cache, instead, created just now by executor

router/httpTriggers.go

@@ -52,10 +52,11 @@ type HTTPTriggerSet struct {
 	recorderSet                *RecorderSet
 	updateRouterRequestChannel chan struct{}
 	tsRoundTripperParams       *tsRoundTripperParams
+	isDebugEnv                 bool
 }
 
 func makeHTTPTriggerSet(fmap *functionServiceMap, frmap *functionRecorderMap, trmap *triggerRecorderMap, fissionClient *crd.FissionClient,
-	kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient *rest.RESTClient, params *tsRoundTripperParams) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
+	kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient *rest.RESTClient, params *tsRoundTripperParams, isDebugEnv bool) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
 	httpTriggerSet := &HTTPTriggerSet{
 		functionServiceMap:         fmap,
 		triggers:                   []crd.HTTPTrigger{},
@@ -65,6 +66,7 @@ func makeHTTPTriggerSet(fmap *functionServiceMap, frmap *functionRecorderMap, tr
 		crdClient:                  crdClient,
 		updateRouterRequestChannel: make(chan struct{}),
 		tsRoundTripperParams:       params,
+		isDebugEnv:                 isDebugEnv,
 	}
 	var tStore, fnStore, rStore k8sCache.Store
 	var tController, fnController k8sCache.Controller
@@ -150,6 +152,7 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 			fnWeightDistributionList: rr.functionWtDistributionList,
 			tsRoundTripperParams:     ts.tsRoundTripperParams,
 			recorderName:             recorderName,
+			isDebugEnv:               ts.isDebugEnv,
 		}
 
 		if rr.resolveResultType == resolveResultSingleFunction {
@@ -197,6 +200,7 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 			executor:             ts.executor,
 			tsRoundTripperParams: ts.tsRoundTripperParams,
 			recorderName:         recorderName,
+			isDebugEnv:           ts.isDebugEnv,
 		}
 		muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name, function.Metadata.Namespace), fh.handler)
 	}

router/router.go

@@ -124,12 +124,17 @@ func Start(port int, executorUrl string) {
 		log.Fatalf("Failed to parse max retry times: %v", err)
 	}
 
+	isDebugEnv, err := strconv.ParseBool(os.Getenv("DEBUG_ENV"))
+	if err != nil {
+		log.Fatalf("Failed to parse DEBUG_ENV: %v", err)
+	}
+
 	triggers, _, fnStore := makeHTTPTriggerSet(fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{
 		timeout:         timeout,
 		timeoutExponent: timeoutExponent,
 		keepAlive:       keepAlive,
 		maxRetries:      maxRetries,
-	})
+	}, isDebugEnv)
 
 	resolver := makeFunctionReferenceResolver(fnStore)
 

router/router_test.go

@@ -57,7 +57,7 @@ func TestRouter(t *testing.T) {
 			timeoutExponent: 2,
 			keepAlive:       30 * time.Second,
 			maxRetries:      10,
-		})
+		}, false)
 	triggerUrl := "/foo"
 	triggers.triggers = append(triggers.triggers,
 		crd.HTTPTrigger{