Retry pod choose if we get terminated or deleted pod from ready pod controller (#2274)

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
2021-12-10 13:45:44 +05:30
parent fe0c1e3683
commit 35a5397a05
2 changed files with 14 additions and 1 deletions
@@ -263,7 +263,13 @@ func (gp *GenericPool) choosePod(ctx context.Context, newLabels map[string]strin
 		pod, err := gp.readyPodLister.Pods(namespace).Get(name)
 		if err != nil {
 			logger.Error("fetching object from store failed", zap.String("key", key), zap.Error(err))
-			return "", nil, err
+			gp.readyPodQueue.Done(key)
+			continue
+		}
+		if utils.IsPodTerminated(pod) {
+			logger.Error("pod is terminated", zap.String("key", key))
+			gp.readyPodQueue.Done(key)
+			continue
 		}
 		if !utils.IsReadyPod(pod) {
 			logger.Warn("pod not ready, pod will be checked again", zap.String("key", key), zap.Duration("delay", expoDelay))