diff --git a/executor/executor.go b/executor/executor.go index 1ddca8f4..c8205102 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -17,6 +17,7 @@ limitations under the License. package executor import ( + "fmt" "log" "net/http" "runtime/debug" @@ -25,6 +26,7 @@ import ( "time" "github.com/dchest/uniuri" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -113,6 +115,12 @@ func (executor *Executor) serveCreateFuncServices() { // get the function service from the cache fsvc, err := executor.fsCache.GetByFunction(m) + + // fsCache return error when the entry does not exist/expire. + // It normally happened if there are multiple requests are + // waiting for the same function and executor failed to cre- + // ate service for function. + err = errors.Wrap(err, fmt.Sprintf("Error getting service for function %v in namespace %v", m.Name, m.Namespace)) req.respChan <- &createFuncServiceResponse{ funcSvc: fsvc, err: err, @@ -145,10 +153,12 @@ func (executor *Executor) createServiceForFunction(meta *metav1.ObjectMeta) (*fs return nil, err } + var fsvc *fscache.FuncSvc + var fsvcErr error + switch executorType { case fission.ExecutorTypeNewdeploy: - fs, err := executor.ndm.GetFuncSvc(meta) - return fs, err + fsvc, fsvcErr = executor.ndm.GetFuncSvc(meta) default: pool, err := executor.gpm.GetPool(env) if err != nil { @@ -157,9 +167,15 @@ func (executor *Executor) createServiceForFunction(meta *metav1.ObjectMeta) (*fs // from GenericPool -> get one function container // (this also adds to the cache) log.Printf("[%v] getting function service from pool", meta.Name) - fsvc, err := pool.GetFuncSvc(meta) - return fsvc, err + fsvc, fsvcErr = pool.GetFuncSvc(meta) } + + if fsvcErr != nil { + fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%v] Error creating service for function", meta.Name)) + log.Print(fsvcErr) + } + + return fsvc, fsvcErr } func (executor *Executor) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment, error) { diff --git a/executor/poolmgr/gp.go b/executor/poolmgr/gp.go index c1f2ba37..5923e4d6 100644 --- a/executor/poolmgr/gp.go +++ b/executor/poolmgr/gp.go @@ -19,7 +19,6 @@ package poolmgr import ( "bytes" "encoding/json" - "errors" "fmt" "log" "math/rand" @@ -32,7 +31,7 @@ import ( "time" "github.com/dchest/uniuri" - + "github.com/pkg/errors" apiv1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -662,16 +661,22 @@ func (gp *GenericPool) waitForReadyPod() error { depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get( gp.deployment.ObjectMeta.Name, metav1.GetOptions{}) if err != nil { - log.Printf("err: %v", err) + err = errors.Wrap(err, fmt.Sprintf( + "Error waiting for ready pod of deployment %v in namespace %v", + gp.deployment.ObjectMeta.Name, gp.namespace)) + log.Print(err) return err } + gp.deployment = depl if gp.deployment.Status.AvailableReplicas > 0 { return nil } if time.Since(startTime) > gp.podReadyTimeout { - return errors.New("timeout: waited too long for pod to be ready") + return errors.Errorf( + "Timeout: waited too long for pod of deployment %v in namespace %v to be ready", + gp.deployment.ObjectMeta.Name, gp.namespace) } time.Sleep(1000 * time.Millisecond) }