Retry pod choose if we get terminated or deleted pod from ready pod controller (#2274)

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/executor/executortype/poolmgr/gp.go

@@ -263,7 +263,13 @@ func (gp *GenericPool) choosePod(ctx context.Context, newLabels map[string]strin
 		pod, err := gp.readyPodLister.Pods(namespace).Get(name)
 		if err != nil {
 			logger.Error("fetching object from store failed", zap.String("key", key), zap.Error(err))
-			return "", nil, err
+			gp.readyPodQueue.Done(key)
+			continue
+		}
+		if utils.IsPodTerminated(pod) {
+			logger.Error("pod is terminated", zap.String("key", key))
+			gp.readyPodQueue.Done(key)
+			continue
 		}
 		if !utils.IsReadyPod(pod) {
 			logger.Warn("pod not ready, pod will be checked again", zap.String("key", key), zap.Duration("delay", expoDelay))

pkg/utils/podutils.go

@@ -48,6 +48,13 @@ func IsReadyPod(pod *v1.Pod) bool {
 	return true
 }
 
+func IsPodTerminated(pod *v1.Pod) bool {
+	if phase := pod.Status.Phase; phase != v1.PodPending && phase != v1.PodRunning && phase != v1.PodUnknown {
+		return true
+	}
+	return false
+}
+
 // PodContainerReadyStatus returns the number of ready containers and total containers present in pod
 func PodContainerReadyStatus(pod *v1.Pod) (readyContainers, noOfContainers int) {