Move packages to proejct/pkg to follow go project folder structure convention (#1190)

pkg/executor/poolmgr/funcwatcher.go

@@ -0,0 +1,213 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package poolmgr
+
+import (
+	"time"
+
+	"github.com/fission/fission/pkg/types"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/utils"
+)
+
+func getIstioServiceLabels(fnName string) map[string]string {
+	return map[string]string{
+		"functionName": fnName,
+	}
+}
+
+func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset, fissionfnNamespace string, istioEnabled bool) (k8sCache.Store, k8sCache.Controller) {
+
+	resyncPeriod := 30 * time.Second
+	lw := k8sCache.NewListWatchFromClient(fissionClient.GetCrdClient(), "functions", metav1.NamespaceAll, fields.Everything())
+
+	funcStore, controller := k8sCache.NewInformer(lw, &fv1.Function{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				fn := obj.(*fv1.Function)
+
+				// Since istio only allows accessing pod through k8s service,
+				// for the functions with executor type "poolmgr" we need to
+				// create a service for sending requests to pod in pool.
+				// Functions with executor type "Newdeploy" is specialized at
+				// pod starts. In this case, just ignore such functions.
+				fnExecutorType := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
+
+				// In some cases, user may not enter the executorType explicitly, for example in his spec.yaml.
+				// we assume it to be of type poolmgr
+				if fnExecutorType != "" && fnExecutorType != fv1.ExecutorTypePoolmgr {
+					return
+				}
+
+				// create or update role-binding
+				envNs := fissionfnNamespace
+				if fn.Spec.Environment.Namespace != metav1.NamespaceDefault {
+					envNs = fn.Spec.Environment.Namespace
+				}
+
+				// TODO : Just bring to your attention during review :
+				// setup rolebinding is tried, if it fails, we dont return. we just log an error and move on, because :
+				// 1. not all functions have secrets and/or configmaps, so things will work without this rolebinding in that case.
+				// 2. on the contrary, when the route is tried, the env fetcher logs will show a 403 forbidden message and same will be relayed to executor.
+				err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.SecretConfigMapGetterRB, fn.Metadata.Namespace, types.SecretConfigMapGetterCR, types.ClusterRole, types.FissionFetcherSA, envNs)
+				if err != nil {
+					gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", types.SecretConfigMapGetterRB))
+				} else {
+					gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
+						zap.String("service_account", types.FissionFetcherSA),
+						zap.String("service_account_namepsace", envNs),
+						zap.String("function_name", fn.Metadata.Name),
+						zap.String("function_namespace", fn.Metadata.Namespace))
+				}
+
+				if istioEnabled {
+					// create a same name service for function
+					// since istio only allows the traffic to service
+					sel := map[string]string{
+						"functionName": fn.Metadata.Name,
+						"functionUid":  string(fn.Metadata.UID),
+					}
+
+					svcName := utils.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
+
+					// service for accepting user traffic
+					svc := apiv1.Service{
+						ObjectMeta: metav1.ObjectMeta{
+							Namespace: envNs,
+							Name:      svcName,
+							Labels:    getIstioServiceLabels(fn.Metadata.Name),
+						},
+						Spec: apiv1.ServiceSpec{
+							Type: apiv1.ServiceTypeClusterIP,
+							Ports: []apiv1.ServicePort{
+								// Service port name should begin with a recognized prefix, or the traffic will be
+								// treated as TCP traffic. (https://istio.io/docs/setup/kubernetes/sidecar-injection.html)
+								// Originally the ports' name are similar to "http-fetch" and "http-specialize".
+								// But for istio 0.5.1, istio-proxy return unexpected 431 error with such naming.
+								// https://github.com/istio/istio/issues/928
+								// Workaround: remove prefix
+								// TODO: prepend prefix once the bug fixed
+								{
+									Name:       "fetch",
+									Protocol:   apiv1.ProtocolTCP,
+									Port:       8000,
+									TargetPort: intstr.FromInt(8000),
+								},
+								{
+									Name:       "specialize",
+									Protocol:   apiv1.ProtocolTCP,
+									Port:       8888,
+									TargetPort: intstr.FromInt(8888),
+								},
+							},
+							Selector: sel,
+						},
+					}
+
+					// create function istio service if it does not exist
+					_, err = kubernetesClient.CoreV1().Services(envNs).Create(&svc)
+					if err != nil && !kerrors.IsAlreadyExists(err) {
+						gpm.logger.Error("error creating istio service for function",
+							zap.Error(err),
+							zap.String("service_name", svcName),
+							zap.String("function_name", fn.Metadata.Name),
+							zap.Any("selectors", sel))
+					}
+				}
+			},
+
+			DeleteFunc: func(obj interface{}) {
+				fn := obj.(*fv1.Function)
+
+				fnExecutorType := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
+				if fnExecutorType != "" && fnExecutorType != fv1.ExecutorTypePoolmgr {
+					return
+				}
+
+				envNs := fissionfnNamespace
+				if fn.Spec.Environment.Namespace != metav1.NamespaceDefault {
+					envNs = fn.Spec.Environment.Namespace
+				}
+
+				if istioEnabled {
+					svcName := utils.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
+					// delete function istio service
+					err := kubernetesClient.CoreV1().Services(envNs).Delete(svcName, nil)
+					if err != nil && !kerrors.IsNotFound(err) {
+						gpm.logger.Error("error deleting istio service for function",
+							zap.Error(err),
+							zap.String("service_name", svcName),
+							zap.String("function_name", fn.Metadata.Name))
+
+					}
+				}
+			},
+
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				oldFunc := oldObj.(*fv1.Function)
+				newFunc := newObj.(*fv1.Function)
+
+				if oldFunc.Metadata.ResourceVersion == newFunc.Metadata.ResourceVersion {
+					return
+				}
+
+				envChanged := (oldFunc.Spec.Environment.Namespace != newFunc.Spec.Environment.Namespace)
+
+				executorTypeChangedToPM := (oldFunc.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypePoolmgr &&
+					newFunc.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypePoolmgr)
+
+				// if a func's env reference gets updated and the newly referenced env is in a different ns,
+				// we need to create a rolebinding in func's ns so that the fetcher-sa in env ns has access
+				// to fetch secrets and config maps from the func's ns.
+				// similarly if executorType changed to Pool Manager, we now need a rolebinding in the func ns for fetcher sa
+				// present in env ns because for newdeploy, the fetcher sa is in function namespace
+				if envChanged || executorTypeChangedToPM {
+					envNs := fissionfnNamespace
+					if newFunc.Spec.Environment.Namespace != metav1.NamespaceDefault {
+						envNs = newFunc.Spec.Environment.Namespace
+					}
+
+					err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.SecretConfigMapGetterRB,
+						newFunc.Metadata.Namespace, types.SecretConfigMapGetterCR, types.ClusterRole,
+						types.FissionFetcherSA, envNs)
+
+					if err != nil {
+						gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", types.SecretConfigMapGetterRB))
+					} else {
+						gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
+							zap.String("service_account", types.FissionFetcherSA),
+							zap.String("service_account_namepsace", envNs),
+							zap.String("function_name", newFunc.Metadata.Name),
+							zap.String("function_namespace", newFunc.Metadata.Namespace))
+					}
+				}
+			},
+		})
+
+	return funcStore, controller
+}

pkg/executor/poolmgr/gp.go

@@ -0,0 +1,626 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package poolmgr
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/dchest/uniuri"
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+	multierror "github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/executor/fscache"
+	"github.com/fission/fission/pkg/executor/util"
+	fetcherClient "github.com/fission/fission/pkg/fetcher/client"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+)
+
+type (
+	GenericPool struct {
+		logger                 *zap.Logger
+		env                    *fv1.Environment
+		replicas               int32                         // num idle pods
+		deployment             *v1beta1.Deployment           // kubernetes deployment
+		namespace              string                        // namespace to keep our resources
+		functionNamespace      string                        // fallback namespace for fission functions
+		podReadyTimeout        time.Duration                 // timeout for generic pods to become ready
+		idlePodReapTime        time.Duration                 // pods unused for idlePodReapTime are deleted
+		fsCache                *fscache.FunctionServiceCache // cache funcSvc's by function, address and podname
+		useSvc                 bool                          // create k8s service for specialized pods
+		useIstio               bool
+		poolInstanceId         string           // small random string to uniquify pod names
+		runtimeImagePullPolicy apiv1.PullPolicy // pull policy for generic pool to created env deployment
+		kubernetesClient       *kubernetes.Clientset
+		fissionClient          *crd.FissionClient
+		instanceId             string // poolmgr instance id
+		labelsForPool          map[string]string
+		requestChannel         chan *choosePodRequest
+		fetcherConfig          *fetcherConfig.Config
+	}
+
+	// serialize the choosing of pods so that choices don't conflict
+	choosePodRequest struct {
+		newLabels       map[string]string
+		responseChannel chan *choosePodResponse
+	}
+	choosePodResponse struct {
+		pod *apiv1.Pod
+		error
+	}
+)
+
+func MakeGenericPool(
+	logger *zap.Logger,
+	fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset,
+	env *fv1.Environment,
+	initialReplicas int32,
+	namespace string,
+	functionNamespace string,
+	fsCache *fscache.FunctionServiceCache,
+	fetcherConfig *fetcherConfig.Config,
+	instanceId string,
+	enableIstio bool) (*GenericPool, error) {
+
+	gpLogger := logger.Named("generic_pool")
+
+	gpLogger.Info("creating pool", zap.Any("environment", env.Metadata))
+
+	// TODO: in general we need to provide the user a way to configure pools.  Initial
+	// replicas, autoscaling params, various timeouts, etc.
+	gp := &GenericPool{
+		logger:            gpLogger,
+		env:               env,
+		replicas:          initialReplicas, // TODO make this an env param instead?
+		requestChannel:    make(chan *choosePodRequest),
+		fissionClient:     fissionClient,
+		kubernetesClient:  kubernetesClient,
+		namespace:         namespace,
+		functionNamespace: functionNamespace,
+		podReadyTimeout:   5 * time.Minute, // TODO make this an env param?
+		idlePodReapTime:   3 * time.Minute, // TODO make this configurable
+		fsCache:           fsCache,
+		poolInstanceId:    uniuri.NewLen(8),
+		fetcherConfig:     fetcherConfig,
+		instanceId:        instanceId,
+		useSvc:            false,       // defaults off -- svc takes a second or more to become routable, slowing cold start
+		useIstio:          enableIstio, // defaults off -- istio integration requires pod relabeling and it takes a second or more to become routable, slowing cold start
+	}
+
+	gp.runtimeImagePullPolicy = utils.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY"))
+
+	// create fetcher SA in this ns, if not already created
+	err := fetcherConfig.SetupServiceAccount(gp.kubernetesClient, gp.namespace, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error creating fetcher service account in namespace %q", gp.namespace)
+	}
+
+	// Labels for generic deployment/RS/pods.
+	gp.labelsForPool = gp.getDeployLabels()
+
+	// create the pool
+	err = gp.createPool()
+	if err != nil {
+		return nil, err
+	}
+	gpLogger.Info("deployment created", zap.Any("environment", env.Metadata))
+
+	go gp.choosePodService()
+
+	return gp, nil
+}
+
+func (gp *GenericPool) getDeployLabels() map[string]string {
+	return map[string]string{
+		fv1.EXECUTOR_INSTANCEID_LABEL: gp.instanceId,
+		types.EXECUTOR_TYPE:           fv1.ExecutorTypePoolmgr,
+		types.ENVIRONMENT_NAME:        gp.env.Metadata.Name,
+		types.ENVIRONMENT_NAMESPACE:   gp.env.Metadata.Namespace,
+		types.ENVIRONMENT_UID:         string(gp.env.Metadata.UID),
+		"managed":                     "true", // this allows us to easily find pods managed by the deployment
+	}
+}
+
+// choosePodService serializes the choosing of pods
+func (gp *GenericPool) choosePodService() {
+	for {
+		select {
+		case req := <-gp.requestChannel:
+			pod, err := gp._choosePod(req.newLabels)
+			if err != nil {
+				req.responseChannel <- &choosePodResponse{error: err}
+				continue
+			}
+			req.responseChannel <- &choosePodResponse{pod: pod}
+		}
+	}
+}
+
+// choosePod picks a ready pod from the pool and relabels it, waiting if necessary.
+// returns the pod API object.
+func (gp *GenericPool) choosePod(newLabels map[string]string) (*apiv1.Pod, error) {
+	req := &choosePodRequest{
+		newLabels:       newLabels,
+		responseChannel: make(chan *choosePodResponse),
+	}
+	gp.requestChannel <- req
+	resp := <-req.responseChannel
+	return resp.pod, resp.error
+}
+
+// _choosePod is called serially by choosePodService
+func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, error) {
+	startTime := time.Now()
+	for {
+		// Retries took too long, error out.
+		if time.Since(startTime) > gp.podReadyTimeout {
+			gp.logger.Error("timed out waiting for pod", zap.Any("labels", newLabels), zap.Duration("timeout", gp.podReadyTimeout))
+			return nil, errors.New("timeout: waited too long to get a ready pod")
+		}
+
+		// Get pods; filter the ones that are ready
+		podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(
+			metav1.ListOptions{
+				LabelSelector: labels.Set(
+					gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
+			})
+		if err != nil {
+			return nil, err
+		}
+		readyPods := make([]*apiv1.Pod, 0, len(podList.Items))
+		for i := range podList.Items {
+			pod := podList.Items[i]
+
+			// Ignore not ready pod here
+			if !utils.IsReadyPod(&pod) {
+				continue
+			}
+
+			// add it to the list of ready pods
+			readyPods = append(readyPods, &pod)
+		}
+		gp.logger.Info("found ready pods",
+			zap.Any("labels", newLabels),
+			zap.Int("ready_count", len(readyPods)),
+			zap.Int("total", len(podList.Items)))
+
+		// If there are no ready pods, wait and retry.
+		if len(readyPods) == 0 {
+			err = gp.waitForReadyPod()
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+
+		// Pick a ready pod.  For now just choose randomly;
+		// ideally we'd care about which node it's running on,
+		// and make a good scheduling decision.
+		chosenPod := readyPods[rand.Intn(len(readyPods))]
+
+		if gp.env.Spec.AllowedFunctionsPerContainer != types.AllowedFunctionsPerContainerInfinite {
+			// Relabel.  If the pod already got picked and
+			// modified, this should fail; in that case just
+			// retry.
+			chosenPod.ObjectMeta.Labels = newLabels
+			gp.logger.Info("relabeling pod", zap.String("pod", chosenPod.ObjectMeta.Name))
+			_, err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Update(chosenPod)
+			if err != nil {
+				gp.logger.Error("failed to relabel pod", zap.Error(err), zap.String("pod", chosenPod.ObjectMeta.Name))
+				continue
+			}
+		}
+		gp.logger.Info("chose pod", zap.String("pod", chosenPod.ObjectMeta.Name), zap.Duration("elapsed_time", time.Since(startTime)))
+		return chosenPod, nil
+	}
+}
+
+func (gp *GenericPool) labelsForFunction(metadata *metav1.ObjectMeta) map[string]string {
+	label := gp.getDeployLabels()
+	label[types.FUNCTION_NAME] = metadata.Name
+	label[types.FUNCTION_UID] = string(metadata.UID)
+	label[types.FUNCTION_NAMESPACE] = metadata.Namespace // function CRD must stay within same namespace of environment CRD
+	label["managed"] = "false"                           // this allows us to easily find pods not managed by the deployment
+	return label
+
+}
+
+func (gp *GenericPool) scheduleDeletePod(name string) {
+	go func() {
+		// The sleep allows debugging or collecting logs from the pod before it's
+		// cleaned up.  (We need a better solutions for both those things; log
+		// aggregation and storage will help.)
+		gp.logger.Error("error in pod - scheduling cleanup", zap.String("pod", name))
+		// Ignore sleep here if istio feature is enabled, function pod
+		// will be deleted after 6 mins (terminationGracePeriodSeconds).
+		if !gp.useIstio {
+			time.Sleep(5 * time.Minute)
+		}
+		gp.kubernetesClient.CoreV1().Pods(gp.namespace).Delete(name, nil)
+	}()
+}
+
+func IsIPv6(podIP string) bool {
+	ip := net.ParseIP(podIP)
+	return ip != nil && strings.Contains(podIP, ":")
+}
+
+func (gp *GenericPool) getSpecializeUrl(podIP string) string {
+	testUrl := os.Getenv("TEST_SPECIALIZE_URL")
+	if len(testUrl) != 0 {
+		// it takes a second or so for the test service to
+		// become routable once a pod is relabeled. This is
+		// super hacky, but only runs in unit tests.
+		time.Sleep(5 * time.Second)
+		return testUrl
+	}
+	isv6 := IsIPv6(podIP)
+	var baseUrl string
+	if isv6 == false {
+		baseUrl = fmt.Sprintf("http://%v:8000/", podIP)
+	} else if isv6 == true { // We use bracket if the IP is in IPv6.
+		baseUrl = fmt.Sprintf("http://[%v]:8000/", podIP)
+	}
+	return baseUrl
+
+}
+
+// specializePod chooses a pod, copies the required user-defined function to that pod
+// (via fetcher), and calls the function-run container to load it, resulting in a
+// specialized pod.
+func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metadata *metav1.ObjectMeta) error {
+	// for fetcher we don't need to create a service, just talk to the pod directly
+	podIP := pod.Status.PodIP
+	if len(podIP) == 0 {
+		return errors.Errorf("Pod %s in namespace %s has no IP", pod.ObjectMeta.Name, pod.ObjectMeta.Namespace)
+	}
+	// specialize pod with service
+	if gp.useIstio {
+		svc := utils.GetFunctionIstioServiceName(metadata.Name, metadata.Namespace)
+		podIP = fmt.Sprintf("%v.%v", svc, gp.namespace)
+	}
+
+	// tell fetcher to get the function.
+	fetcherUrl := gp.getSpecializeUrl(podIP)
+	gp.logger.Info("calling fetcher to copy function", zap.String("function", metadata.Name), zap.String("url", fetcherUrl))
+
+	fn, err := gp.fissionClient.
+		Functions(metadata.Namespace).
+		Get(metadata.Name)
+	if err != nil {
+		return err
+	}
+
+	specializeReq := gp.fetcherConfig.NewSpecializeRequest(fn, gp.env)
+
+	gp.logger.Info("specializing pod", zap.String("function", metadata.Name))
+
+	err = fetcherClient.MakeClient(gp.logger, fetcherUrl).Specialize(ctx, &specializeReq)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// getPoolName returns a unique name of an environment
+func (gp *GenericPool) getPoolName() string {
+	return strings.ToLower(fmt.Sprintf("poolmgr-%v-%v-%v", gp.env.Metadata.Name, gp.env.Metadata.Namespace, uniuri.NewLen(8)))
+}
+
+// A pool is a deployment of generic containers for an env.  This
+// creates the pool but doesn't wait for any pods to be ready.
+func (gp *GenericPool) createPool() error {
+	// Use long terminationGracePeriodSeconds for connection draining in case that
+	// pod still runs user functions.
+	gracePeriodSeconds := int64(6 * 60)
+	if gp.env.Spec.TerminationGracePeriod > 0 {
+		gracePeriodSeconds = gp.env.Spec.TerminationGracePeriod
+	}
+
+	podAnnotations := gp.env.Metadata.Annotations
+	if podAnnotations == nil {
+		podAnnotations = make(map[string]string)
+	}
+	if gp.useIstio && gp.env.Spec.AllowAccessToExternalNetwork {
+		podAnnotations["sidecar.istio.io/inject"] = "false"
+	}
+
+	deployment := &v1beta1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   gp.getPoolName(),
+			Labels: gp.labelsForPool,
+		},
+		Spec: v1beta1.DeploymentSpec{
+			Replicas: &gp.replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: gp.labelsForPool,
+			},
+			Template: apiv1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      gp.labelsForPool,
+					Annotations: podAnnotations,
+				},
+				Spec: apiv1.PodSpec{
+					Containers: []apiv1.Container{
+						util.MergeContainerSpecs(&apiv1.Container{
+							Name:                   gp.env.Metadata.Name,
+							Image:                  gp.env.Spec.Runtime.Image,
+							ImagePullPolicy:        gp.runtimeImagePullPolicy,
+							TerminationMessagePath: "/dev/termination-log",
+							Resources:              gp.env.Spec.Resources,
+							// Pod is removed from endpoints list for service when it's
+							// state became "Termination". We used preStop hook as the
+							// workaround for connection draining since pod maybe shutdown
+							// before grace period expires.
+							// https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods
+							// https://github.com/kubernetes/kubernetes/issues/47576#issuecomment-308900172
+							Lifecycle: &apiv1.Lifecycle{
+								PreStop: &apiv1.Handler{
+									Exec: &apiv1.ExecAction{
+										Command: []string{
+											"/bin/sleep",
+											fmt.Sprintf("%v", gracePeriodSeconds),
+										},
+									},
+								},
+							},
+						}, gp.env.Spec.Runtime.Container),
+					},
+					ServiceAccountName: "fission-fetcher",
+					// TerminationGracePeriodSeconds should be equal to the
+					// sleep time of preStop to make sure that SIGTERM is sent
+					// to pod after 6 mins.
+					TerminationGracePeriodSeconds: &gracePeriodSeconds,
+				},
+			},
+		},
+	}
+
+	// Order of merging is important here - first fetcher, then containers and lastly pod spec
+	err := gp.fetcherConfig.AddFetcherToPodSpec(&deployment.Spec.Template.Spec, gp.env.Metadata.Name)
+	if err != nil {
+		return err
+	}
+
+	if gp.env.Spec.Runtime.PodSpec != nil {
+		err = util.MergePodSpec(&deployment.Spec.Template.Spec, gp.env.Spec.Runtime.PodSpec)
+		if err != nil {
+			return err
+		}
+	}
+
+	depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment)
+	if err != nil {
+		gp.logger.Error("error creating deployment in kubernetes", zap.Error(err), zap.String("deployment", deployment.Name))
+		return err
+	}
+	gp.deployment = depl
+	return nil
+}
+
+func (gp *GenericPool) waitForReadyPod() error {
+	startTime := time.Now()
+	for {
+		// TODO: for now we just poll; use a watch instead
+		depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get(
+			gp.deployment.ObjectMeta.Name, metav1.GetOptions{})
+		if err != nil {
+			e := "error waiting for ready pod for deployment"
+			gp.logger.Error(e, zap.String("deployment", gp.deployment.ObjectMeta.Name), zap.String("namespace", gp.namespace))
+			return fmt.Errorf("%s %q in namespace %q", e, gp.deployment.ObjectMeta.Name, gp.namespace)
+		}
+
+		gp.deployment = depl
+		if gp.deployment.Status.AvailableReplicas > 0 {
+			return nil
+		}
+
+		if time.Since(startTime) > gp.podReadyTimeout {
+			podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(metav1.ListOptions{
+				LabelSelector: labels.Set(
+					gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
+			})
+			if err != nil {
+				gp.logger.Error("error getting pod list after timeout waiting for ready pod", zap.Error(err))
+			}
+
+			// Since even single pod is not ready, choosing the first pod to inspect is a good approximation. In future this can be done better
+			pod := podList.Items[0]
+			var multierr *multierror.Error
+			for _, cStatus := range pod.Status.ContainerStatuses {
+				if cStatus.Ready != true {
+					multierr = multierror.Append(multierr, errors.New(fmt.Sprintf("%v: %v", cStatus.State.Waiting.Reason, cStatus.State.Waiting.Message)))
+				}
+			}
+			return errors.Wrapf(multierr, "Timeout: waited too long for pod of deployment %v in namespace %v to be ready",
+				gp.deployment.ObjectMeta.Name, gp.namespace)
+		}
+		time.Sleep(1000 * time.Millisecond)
+	}
+}
+
+func (gp *GenericPool) createSvc(name string, labels map[string]string) (*apiv1.Service, error) {
+	service := apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: apiv1.ServiceSpec{
+			Type: apiv1.ServiceTypeClusterIP,
+			Ports: []apiv1.ServicePort{
+				{
+					Protocol:   apiv1.ProtocolTCP,
+					Port:       80,
+					TargetPort: intstr.FromInt(8888),
+				},
+			},
+			Selector: labels,
+		},
+	}
+	svc, err := gp.kubernetesClient.CoreV1().Services(gp.namespace).Create(&service)
+	return svc, err
+}
+
+func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
+	gp.logger.Info("choosing pod from pool", zap.String("function", m.Name))
+	newLabels := gp.labelsForFunction(m)
+
+	if gp.useIstio {
+		// Istio only allows accessing pod through k8s service, and requests come to
+		// service are not always being routed to the same pod. For example:
+
+		// If there is only one pod (podA) behind the service svcX.
+
+		// svcX -> podA
+
+		// All requests (specialize request & function access requests)
+		// will be routed to podA without any problem.
+
+		// If podA and podB are behind svcX.
+
+		// svcX -> podA (specialized)
+		//      -> podB (non-specialized)
+
+		// The specialize request may be routed to podA and the function access
+		// requests may go to podB. In this case, the function cannot be served
+		// properly.
+
+		// To prevent such problem, we need to delete old versions function pods
+		// and make sure that there is only one pod behind the service
+
+		sel := map[string]string{
+			"functionName": m.Name,
+			"functionUid":  string(m.UID),
+		}
+		podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(metav1.ListOptions{
+			LabelSelector: labels.Set(sel).AsSelector().String(),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// Remove old versions function pods
+		for _, pod := range podList.Items {
+			// Delete pod no matter what status it is
+			gp.kubernetesClient.CoreV1().Pods(gp.namespace).Delete(pod.ObjectMeta.Name, nil)
+		}
+	}
+
+	pod, err := gp.choosePod(newLabels)
+	if err != nil {
+		return nil, err
+	}
+
+	err = gp.specializePod(ctx, pod, m)
+	if err != nil {
+		gp.scheduleDeletePod(pod.ObjectMeta.Name)
+		return nil, err
+	}
+	gp.logger.Info("specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
+
+	var svcHost string
+	if gp.useSvc && !gp.useIstio {
+		svcName := fmt.Sprintf("svc-%v", m.Name)
+		if len(m.UID) > 0 {
+			svcName = fmt.Sprintf("%s-%v", svcName, m.UID)
+		}
+
+		labels := gp.labelsForFunction(m)
+		svc, err := gp.createSvc(svcName, labels)
+		if err != nil {
+			gp.scheduleDeletePod(pod.ObjectMeta.Name)
+			return nil, err
+		}
+		if svc.ObjectMeta.Name != svcName {
+			gp.scheduleDeletePod(pod.ObjectMeta.Name)
+			return nil, errors.Errorf("sanity check failed for svc %v", svc.ObjectMeta.Name)
+		}
+
+		// the fission router isn't in the same namespace, so return a
+		// namespace-qualified hostname
+		svcHost = fmt.Sprintf("%v.%v", svcName, gp.namespace)
+	} else if gp.useIstio {
+		svc := utils.GetFunctionIstioServiceName(m.Name, m.Namespace)
+		svcHost = fmt.Sprintf("%v.%v:8888", svc, gp.namespace)
+	} else {
+		gp.logger.Info("using pod IP for specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
+		svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP)
+	}
+
+	kubeObjRefs := []apiv1.ObjectReference{
+		{
+			Kind:            "pod",
+			Name:            pod.ObjectMeta.Name,
+			APIVersion:      pod.TypeMeta.APIVersion,
+			Namespace:       pod.ObjectMeta.Namespace,
+			ResourceVersion: pod.ObjectMeta.ResourceVersion,
+			UID:             pod.ObjectMeta.UID,
+		},
+	}
+
+	fsvc := &fscache.FuncSvc{
+		Name:              pod.ObjectMeta.Name,
+		Function:          m,
+		Environment:       gp.env,
+		Address:           svcHost,
+		KubernetesObjects: kubeObjRefs,
+		Executor:          fscache.POOLMGR,
+		Ctime:             time.Now(),
+		Atime:             time.Now(),
+	}
+
+	_, err = gp.fsCache.Add(*fsvc)
+	if err != nil {
+		return nil, err
+	}
+	return fsvc, nil
+}
+
+// destroys the pool -- the deployment, replicaset and pods
+func (gp *GenericPool) destroy() error {
+	deletePropagation := metav1.DeletePropagationBackground
+	delOpt := metav1.DeleteOptions{
+		PropagationPolicy: &deletePropagation,
+	}
+	err := gp.kubernetesClient.ExtensionsV1beta1().
+		Deployments(gp.namespace).Delete(gp.deployment.ObjectMeta.Name, &delOpt)
+	if err != nil {
+		gp.logger.Error("error destroying deployment",
+			zap.Error(err),
+			zap.String("deployment_name", gp.deployment.ObjectMeta.Name),
+			zap.String("deployment_namespace", gp.namespace))
+		return err
+	}
+	return nil
+}

pkg/executor/poolmgr/gpm.go

@@ -0,0 +1,368 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package poolmgr
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/fission/fission/pkg/utils"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sTypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/cache"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/executor/fscache"
+	"github.com/fission/fission/pkg/executor/reaper"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+	"github.com/fission/fission/pkg/types"
+)
+
+type requestType int
+
+const (
+	GET_POOL requestType = iota
+	CLEANUP_POOLS
+)
+
+type (
+	GenericPoolManager struct {
+		logger *zap.Logger
+
+		pools            map[string]*GenericPool
+		kubernetesClient *kubernetes.Clientset
+		namespace        string
+
+		fissionClient  *crd.FissionClient
+		functionEnv    *cache.Cache
+		fsCache        *fscache.FunctionServiceCache
+		instanceId     string
+		requestChannel chan *request
+
+		enableIstio   bool
+		fetcherConfig *fetcherConfig.Config
+
+		funcStore      k8sCache.Store
+		funcController k8sCache.Controller
+		pkgStore       k8sCache.Store
+		pkgController  k8sCache.Controller
+
+		idlePodReapTime time.Duration
+	}
+	request struct {
+		requestType
+		env             *fv1.Environment
+		envList         []fv1.Environment
+		responseChannel chan *response
+	}
+	response struct {
+		error
+		pool *GenericPool
+	}
+)
+
+func MakeGenericPoolManager(
+	logger *zap.Logger,
+	fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset,
+	functionNamespace string,
+	fetcherConfig *fetcherConfig.Config,
+	instanceId string) *GenericPoolManager {
+
+	gpmLogger := logger.Named("generic_pool_manager")
+
+	gpm := &GenericPoolManager{
+		logger:           gpmLogger,
+		pools:            make(map[string]*GenericPool),
+		kubernetesClient: kubernetesClient,
+		namespace:        functionNamespace,
+		fissionClient:    fissionClient,
+		functionEnv:      cache.MakeCache(10*time.Second, 0),
+		fsCache:          fscache.MakeFunctionServiceCache(gpmLogger),
+		instanceId:       instanceId,
+		requestChannel:   make(chan *request),
+		idlePodReapTime:  2 * time.Minute,
+		fetcherConfig:    fetcherConfig,
+	}
+	go gpm.service()
+	go gpm.eagerPoolCreator()
+
+	if len(os.Getenv("ENABLE_ISTIO")) > 0 {
+		istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
+		if err != nil {
+			gpmLogger.Info("failed to parse ENABLE_ISTIO")
+		}
+		gpm.enableIstio = istio
+	}
+
+	gpm.funcStore, gpm.funcController = gpm.makeFuncController(
+		gpm.fissionClient, gpm.kubernetesClient, gpm.namespace, gpm.enableIstio)
+
+	gpm.pkgStore, gpm.pkgController = gpm.makePkgController(gpm.fissionClient, gpm.kubernetesClient, gpm.namespace)
+
+	return gpm
+}
+
+func (gpm *GenericPoolManager) Run(ctx context.Context) {
+	go gpm.funcController.Run(ctx.Done())
+	go gpm.pkgController.Run(ctx.Done())
+	go gpm.idleObjectReaper()
+}
+
+func (gpm *GenericPoolManager) service() {
+	for {
+		req := <-gpm.requestChannel
+		switch req.requestType {
+		case GET_POOL:
+			// just because they are missing in the cache, we end up creating another duplicate pool.
+			var err error
+			pool, ok := gpm.pools[crd.CacheKey(&req.env.Metadata)]
+			if !ok {
+				poolsize := gpm.getEnvPoolsize(req.env)
+				switch req.env.Spec.AllowedFunctionsPerContainer {
+				case types.AllowedFunctionsPerContainerInfinite:
+					poolsize = 1
+				}
+
+				// To support backward compatibility, if envs are created in default ns, we go ahead
+				// and create pools in fission-function ns as earlier.
+				ns := gpm.namespace
+				if req.env.Metadata.Namespace != metav1.NamespaceDefault {
+					ns = req.env.Metadata.Namespace
+				}
+
+				pool, err = MakeGenericPool(gpm.logger,
+					gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize,
+					ns, gpm.namespace, gpm.fsCache, gpm.fetcherConfig, gpm.instanceId, gpm.enableIstio)
+				if err != nil {
+					req.responseChannel <- &response{error: err}
+					continue
+				}
+				gpm.pools[crd.CacheKey(&req.env.Metadata)] = pool
+			}
+			req.responseChannel <- &response{pool: pool}
+		case CLEANUP_POOLS:
+			latestEnvPoolsize := make(map[string]int)
+			for _, env := range req.envList {
+				latestEnvPoolsize[crd.CacheKey(&env.Metadata)] = int(gpm.getEnvPoolsize(&env))
+			}
+			for key, pool := range gpm.pools {
+				poolsize, ok := latestEnvPoolsize[key]
+				if !ok || poolsize == 0 {
+					// Env no longer exists or pool size changed to zero
+
+					gpm.logger.Info("destroying generic pool", zap.Any("environment", pool.env.Metadata))
+					delete(gpm.pools, key)
+
+					// and delete the pool asynchronously.
+					go pool.destroy()
+				}
+			}
+			// no response, caller doesn't wait
+		}
+	}
+}
+
+func (gpm *GenericPoolManager) GetPool(env *fv1.Environment) (*GenericPool, error) {
+	c := make(chan *response)
+	gpm.requestChannel <- &request{
+		requestType:     GET_POOL,
+		env:             env,
+		responseChannel: c,
+	}
+	resp := <-c
+	return resp.pool, resp.error
+}
+
+func (gpm *GenericPoolManager) CleanupPools(envs []fv1.Environment) {
+	gpm.requestChannel <- &request{
+		requestType: CLEANUP_POOLS,
+		envList:     envs,
+	}
+}
+
+func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
+	// from Func -> get Env
+	gpm.logger.Info("getting environment for function", zap.String("function", metadata.Name))
+	env, err := gpm.getFunctionEnv(metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	pool, err := gpm.GetPool(env)
+	if err != nil {
+		return nil, err
+	}
+	// from GenericPool -> get one function container
+	// (this also adds to the cache)
+	gpm.logger.Info("getting function service from pool", zap.String("function", metadata.Name))
+	return pool.GetFuncSvc(ctx, metadata)
+}
+
+func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*fv1.Environment, error) {
+	var env *fv1.Environment
+
+	// Cached ?
+	result, err := gpm.functionEnv.Get(crd.CacheKey(m))
+	if err == nil {
+		env = result.(*fv1.Environment)
+		return env, nil
+	}
+
+	// Cache miss -- get func from controller
+	f, err := gpm.fissionClient.Functions(m.Namespace).Get(m.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get env from metadata
+	gpm.logger.Info("getting env", zap.Any("function", m))
+	env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// cache for future lookups
+	gpm.functionEnv.Set(crd.CacheKey(m), env)
+
+	return env, nil
+}
+
+func (gpm *GenericPoolManager) eagerPoolCreator() {
+	pollSleep := time.Duration(2 * time.Second)
+	for {
+		// get list of envs from controller
+		envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
+		if err != nil {
+			if utils.IsNetworkError(err) {
+				gpm.logger.Error("encountered network error, retrying", zap.Error(err))
+				time.Sleep(5 * time.Second)
+				continue
+			}
+			gpm.logger.Fatal("failed to get environment list", zap.Error(err))
+		}
+
+		// Create pools for all envs.  TODO: we should make this a bit less eager, only
+		// creating pools for envs that are actually used by functions.  Also we might want
+		// to keep these eagerly created pools smaller than the ones created when there are
+		// actual function calls.
+		for i := range envs.Items {
+			env := envs.Items[i]
+			// Create pool only if poolsize greater than zero
+			if gpm.getEnvPoolsize(&env) > 0 {
+				_, err := gpm.GetPool(&envs.Items[i])
+				if err != nil {
+					gpm.logger.Error("eager-create pool failed", zap.Error(err))
+				}
+			}
+		}
+
+		// Clean up pools whose env was deleted
+		gpm.CleanupPools(envs.Items)
+		time.Sleep(pollSleep)
+	}
+}
+
+func (gpm *GenericPoolManager) getEnvPoolsize(env *fv1.Environment) int32 {
+	var poolsize int32
+	if env.Spec.Version < 3 {
+		poolsize = 3
+	} else {
+		poolsize = int32(env.Spec.Poolsize)
+	}
+	return poolsize
+}
+
+// IsValid checks if pod is not deleted and that it has the address passed as the argument. Also checks that all the
+// containers in it are reporting a ready status for the healthCheck.
+func (gpm *GenericPoolManager) IsValid(fsvc *fscache.FuncSvc) bool {
+	for _, obj := range fsvc.KubernetesObjects {
+		if obj.Kind == "pod" {
+			pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{})
+			if err == nil && strings.Contains(fsvc.Address, pod.Status.PodIP) && utils.IsReadyPod(pod) {
+				gpm.logger.Info("valid pod address", zap.String("address", fsvc.Address))
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// idleObjectReaper reaps objects after certain idle time
+func (gpm *GenericPoolManager) idleObjectReaper() {
+
+	pollSleep := time.Duration(gpm.idlePodReapTime)
+	for {
+		time.Sleep(pollSleep)
+
+		envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
+		if err != nil {
+			gpm.logger.Fatal("failed to get environment list", zap.Error(err))
+		}
+
+		envList := make(map[k8sTypes.UID]struct{})
+		for _, env := range envs.Items {
+			envList[env.Metadata.UID] = struct{}{}
+		}
+
+		funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime)
+		if err != nil {
+			gpm.logger.Error("error reaping idle pods", zap.Error(err))
+			continue
+		}
+
+		for _, fsvc := range funcSvcs {
+			if fsvc.Executor != fscache.POOLMGR {
+				continue
+			}
+
+			// For function with the environment that no longer exists, executor
+			// cleanups the idle pod as usual and prints log to notify user.
+			if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
+				gpm.logger.Info("function environment no longer exists",
+					zap.String("environment", fsvc.Environment.Metadata.Name),
+					zap.String("function", fsvc.Name))
+			}
+
+			if fsvc.Environment.Spec.AllowedFunctionsPerContainer == types.AllowedFunctionsPerContainerInfinite {
+				continue
+			}
+
+			deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime)
+			if err != nil {
+				gpm.logger.Error("error deleting Kubernetes objects for function service",
+					zap.Error(err),
+					zap.Any("service", fsvc))
+			}
+
+			if !deleted {
+				continue
+			}
+
+			for _, kubeobj := range fsvc.KubernetesObjects {
+				reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &kubeobj)
+			}
+		}
+	}
+}

pkg/executor/poolmgr/packagewatcher.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package poolmgr
+
+import (
+	"time"
+
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+)
+
+// TODO : It may make sense to make each of add, update, delete funcs run as separate go routines.
+func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset, fissionfnNamespace string) (k8sCache.Store, k8sCache.Controller) {
+
+	resyncPeriod := 30 * time.Second
+	lw := k8sCache.NewListWatchFromClient(fissionClient.GetCrdClient(), "packages", metav1.NamespaceAll, fields.Everything())
+	pkgStore, controller := k8sCache.NewInformer(lw, &fv1.Package{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				pkg := obj.(*fv1.Package)
+				gpm.logger.Debug("list watch for package reported a new package addition",
+					zap.String("package_name", pkg.Metadata.Name),
+					zap.String("package_namepsace", pkg.Metadata.Namespace))
+
+				// create or update role-binding for fetcher sa in env ns to be able to get the pkg contents from pkg namespace
+				envNs := fissionfnNamespace
+				if pkg.Spec.Environment.Namespace != metav1.NamespaceDefault {
+					envNs = pkg.Spec.Environment.Namespace
+				}
+
+				// here, we return if we hit an error during rolebinding setup. this is because this rolebinding is mandatory for
+				// every function's package to be loaded into its env. without that, there's no point to move forward.
+				err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.PackageGetterRB, pkg.Metadata.Namespace, types.PackageGetterCR, types.ClusterRole, types.FissionFetcherSA, envNs)
+				if err != nil {
+					gpm.logger.Error("error creating rolebinding for package",
+						zap.Error(err),
+						zap.String("role_binding", types.PackageGetterRB),
+						zap.String("package_name", pkg.Metadata.Name),
+						zap.String("package_namespace", pkg.Metadata.Namespace))
+					return
+				}
+
+				gpm.logger.Debug("successfully set up rolebinding for fetcher service account",
+					zap.String("service_account", types.FissionFetcherSA),
+					zap.String("service_account_namespace", envNs),
+					zap.String("package_name", pkg.Metadata.Name),
+					zap.String("package_namespace", pkg.Metadata.Namespace))
+			},
+
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				oldPkg := oldObj.(*fv1.Package)
+				newPkg := newObj.(*fv1.Package)
+
+				if oldPkg.Metadata.ResourceVersion == newPkg.Metadata.ResourceVersion {
+					return
+				}
+
+				// if a pkg's env reference gets updated and the newly referenced env is in a different ns,
+				// we need to update the role-binding in pkg ns to grant permissions to the fetcher-sa in env ns
+				// to do a get on pkg
+				if oldPkg.Spec.Environment.Namespace != newPkg.Spec.Environment.Namespace {
+					envNs := fissionfnNamespace
+					if newPkg.Spec.Environment.Namespace != metav1.NamespaceDefault {
+						envNs = newPkg.Spec.Environment.Namespace
+					}
+
+					err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.PackageGetterRB,
+						newPkg.Metadata.Namespace, types.PackageGetterCR, types.ClusterRole,
+						types.FissionFetcherSA, envNs)
+					if err != nil {
+						gpm.logger.Error("error updating rolebinding for package",
+							zap.Error(err),
+							zap.String("role_binding", types.PackageGetterRB),
+							zap.String("package_name", newPkg.Metadata.Name),
+							zap.String("package_namespace", newPkg.Metadata.Namespace))
+						return
+					}
+
+					gpm.logger.Debug("successfully updated rolebinding for fetcher service account",
+						zap.String("service_account", types.FissionFetcherSA),
+						zap.String("service_account_namespace", envNs),
+						zap.String("package_name", newPkg.Metadata.Name),
+						zap.String("package_namespace", newPkg.Metadata.Namespace))
+				}
+			},
+		})
+
+	return pkgStore, controller
+}