Check pod container ready state (#861)

Ensure pod is ready by checking pod’s containers state and deletionTimestamp

common.go

@@ -108,13 +108,20 @@ func IsNetworkDialError(err error) bool {
 	return false
 }
 
-// IsReadyPod checks that all containers in a pod are ready and returns true if so
+// IsReadyPod checks both all containers in a pod are ready and whether
+// the .metadata.DeletionTimestamp is nil.
 func IsReadyPod(pod *apiv1.Pod) bool {
 	// since its a utility function, just ensuring there is no nil pointer exception
 	if pod == nil {
 		return false
 	}
 
+	// pod is in "Terminating" status if deletionTimestamp is not nil
+	// https://github.com/kubernetes/kubernetes/issues/61376
+	if pod.ObjectMeta.DeletionTimestamp != nil {
+		return false
+	}
+
 	for _, cStatus := range pod.Status.ContainerStatuses {
 		if !cStatus.Ready {
 			return false

executor/poolmgr/gp.go

@@ -47,8 +47,6 @@ import (
 	"github.com/fission/fission/executor/util"
 )
 
-const POD_PHASE_RUNNING string = "Running"
-
 type (
 	GenericPool struct {
 		env                    *crd.Environment
@@ -228,21 +226,13 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
 		for i := range podList.Items {
 			pod := podList.Items[i]
 
-			// If a pod has no IP it's not ready
+			// Ignore not ready pod here
-			if len(pod.Status.PodIP) == 0 || string(pod.Status.Phase) != POD_PHASE_RUNNING {
+			if !fission.IsReadyPod(&pod) {
 				continue
 			}
 
-			// Wait for all containers in the pod to be ready
-			podReady := true
-			for _, cs := range pod.Status.ContainerStatuses {
-				podReady = podReady && cs.Ready
-			}
-
 			// add it to the list of ready pods
-			if podReady {
+			readyPods = append(readyPods, &pod)
-				readyPods = append(readyPods, &pod)
-			}
 		}
 		log.Printf("[%v] found %v ready pods of %v total", newLabels, len(readyPods), len(podList.Items))