Add more meaningful error messages to executor when getServiceForFunction (#752)

executor/executor.go

@@ -17,6 +17,7 @@ limitations under the License.
 package executor
 
 import (
+	"fmt"
 	"log"
 	"net/http"
 	"runtime/debug"
@@ -25,6 +26,7 @@ import (
 	"time"
 
 	"github.com/dchest/uniuri"
+	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
@@ -113,6 +115,12 @@ func (executor *Executor) serveCreateFuncServices() {
 
 				// get the function service from the cache
 				fsvc, err := executor.fsCache.GetByFunction(m)
+
+				// fsCache return error when the entry does not exist/expire.
+				// It normally happened if there are multiple requests are
+				// waiting for the same function and executor failed to cre-
+				// ate service for function.
+				err = errors.Wrap(err, fmt.Sprintf("Error getting service for function %v in namespace %v", m.Name, m.Namespace))
 				req.respChan <- &createFuncServiceResponse{
 					funcSvc: fsvc,
 					err:     err,
@@ -145,10 +153,12 @@ func (executor *Executor) createServiceForFunction(meta *metav1.ObjectMeta) (*fs
 		return nil, err
 	}
 
+	var fsvc *fscache.FuncSvc
+	var fsvcErr error
+
 	switch executorType {
 	case fission.ExecutorTypeNewdeploy:
-		fs, err := executor.ndm.GetFuncSvc(meta)
-		return fs, err
+		fsvc, fsvcErr = executor.ndm.GetFuncSvc(meta)
 	default:
 		pool, err := executor.gpm.GetPool(env)
 		if err != nil {
@@ -157,9 +167,15 @@ func (executor *Executor) createServiceForFunction(meta *metav1.ObjectMeta) (*fs
 		// from GenericPool -> get one function container
 		// (this also adds to the cache)
 		log.Printf("[%v] getting function service from pool", meta.Name)
-		fsvc, err := pool.GetFuncSvc(meta)
-		return fsvc, err
+		fsvc, fsvcErr = pool.GetFuncSvc(meta)
 	}
+
+	if fsvcErr != nil {
+		fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%v] Error creating service for function", meta.Name))
+		log.Print(fsvcErr)
+	}
+
+	return fsvc, fsvcErr
 }
 
 func (executor *Executor) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment, error) {

executor/poolmgr/gp.go

@@ -19,7 +19,6 @@ package poolmgr
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"log"
 	"math/rand"
@@ -32,7 +31,7 @@ import (
 	"time"
 
 	"github.com/dchest/uniuri"
-
+	"github.com/pkg/errors"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/api/extensions/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -662,16 +661,22 @@ func (gp *GenericPool) waitForReadyPod() error {
 		depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get(
 			gp.deployment.ObjectMeta.Name, metav1.GetOptions{})
 		if err != nil {
-			log.Printf("err: %v", err)
+			err = errors.Wrap(err, fmt.Sprintf(
+				"Error waiting for ready pod of deployment %v in namespace %v",
+				gp.deployment.ObjectMeta.Name, gp.namespace))
+			log.Print(err)
 			return err
 		}
+
 		gp.deployment = depl
 		if gp.deployment.Status.AvailableReplicas > 0 {
 			return nil
 		}
 
 		if time.Since(startTime) > gp.podReadyTimeout {
-			return errors.New("timeout: waited too long for pod to be ready")
+			return errors.Errorf(
+				"Timeout: waited too long for pod of deployment %v in namespace %v to be ready",
+				gp.deployment.ObjectMeta.Name, gp.namespace)
 		}
 		time.Sleep(1000 * time.Millisecond)
 	}