From 35a5397a054159b00a4a433466de0dc7bbedf492 Mon Sep 17 00:00:00 2001
From: Sanket Sudake <sanketsudake@gmail.com>
Date: Fri, 10 Dec 2021 13:45:44 +0530
Subject: [PATCH] Retry pod choose if we get terminated or deleted pod from
 ready pod controller (#2274)

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
---
 pkg/executor/executortype/poolmgr/gp.go | 8 +++++++-
 pkg/utils/podutils.go                   | 7 +++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/pkg/executor/executortype/poolmgr/gp.go b/pkg/executor/executortype/poolmgr/gp.go
index 794ef69a..4071ac6b 100644
--- a/pkg/executor/executortype/poolmgr/gp.go
+++ b/pkg/executor/executortype/poolmgr/gp.go
@@ -263,7 +263,13 @@ func (gp *GenericPool) choosePod(ctx context.Context, newLabels map[string]strin
 		pod, err := gp.readyPodLister.Pods(namespace).Get(name)
 		if err != nil {
 			logger.Error("fetching object from store failed", zap.String("key", key), zap.Error(err))
-			return "", nil, err
+			gp.readyPodQueue.Done(key)
+			continue
+		}
+		if utils.IsPodTerminated(pod) {
+			logger.Error("pod is terminated", zap.String("key", key))
+			gp.readyPodQueue.Done(key)
+			continue
 		}
 		if !utils.IsReadyPod(pod) {
 			logger.Warn("pod not ready, pod will be checked again", zap.String("key", key), zap.Duration("delay", expoDelay))
diff --git a/pkg/utils/podutils.go b/pkg/utils/podutils.go
index f15ad5f1..d8e56d41 100644
--- a/pkg/utils/podutils.go
+++ b/pkg/utils/podutils.go
@@ -48,6 +48,13 @@ func IsReadyPod(pod *v1.Pod) bool {
 	return true
 }
 
+func IsPodTerminated(pod *v1.Pod) bool {
+	if phase := pod.Status.Phase; phase != v1.PodPending && phase != v1.PodRunning && phase != v1.PodUnknown {
+		return true
+	}
+	return false
+}
+
 // PodContainerReadyStatus returns the number of ready containers and total containers present in pod
 func PodContainerReadyStatus(pod *v1.Pod) (readyContainers, noOfContainers int) {