Invalidate stale router cache entry with podIP's for deleted pods. (#546)

The router's cache entry for a function might become stale if the pod that had the function specialized gets deleted somehow. In such a case, we'd retry getting a new service for the function from executor and retry forwarding the user request to the newly created service.

executor/poolmgr/gpm.go

@@ -21,10 +21,12 @@ import (
 	"log"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/api"
 	k8sCache "k8s.io/client-go/tools/cache"
 
 	"github.com/fission/fission"
@@ -217,3 +219,18 @@ func (gpm *GenericPoolManager) getEnvPoolsize(env *crd.Environment) int32 {
 	}
 	return poolsize
 }
+
+// IsValidPod checks if pod is not deleted and that it has the address passed as the argument. Also checks that all the
+// containers in it are reporting a ready status for the healthCheck.
+func (gpm *GenericPoolManager) IsValidPod(kubeObjects []api.ObjectReference, podAddress string) bool {
+	for _, obj := range kubeObjects {
+		if obj.Kind == "pod" {
+			pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{})
+			if err == nil && strings.Contains(podAddress, pod.Status.PodIP) && fission.IsReadyPod(pod) {
+				log.Printf("Valid pod address : %s", podAddress)
+				return true
+			}
+		}
+	}
+	return false
+}