Enabling multi-tenancy for fission objects. (#655)

This feature allows creation of fission objects in different namespaces, in addition to retaining the existing behavior of creating fission objects in default namespace if user doesnt provide one. 
It also removes cluster admin roles for fission-fetcher and fission-builder Service Accounts and grants them only those privileges that they need.

executor/cleanup.go

@@ -263,3 +263,135 @@ func logErr(msg string, err error) {
 		log.Printf("Error %v: %v", msg, err)
 	}
 }
+
+// cleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or
+// deletes the rolebindings completely if there are no Service Accounts in a rolebinding object.
+func cleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
+	for {
+		log.Println("Starting cleanupRoleBindings cycle")
+		// get all rolebindings ( just to be efficient, one call to kubernetes )
+		rbList, err := client.RbacV1beta1().RoleBindings(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
+		if err != nil {
+			// something wrong, but next iteration hopefully succeeds
+			log.Printf("Error listing rolebindings in all ns, err: %v", err)
+			continue
+		}
+
+		// go through each role-binding object and do the cleanup necessary
+		for _, roleBinding := range rbList.Items {
+			// ignore role-bindings in kube-system namespace
+			if roleBinding.Namespace == "kube-system" {
+				continue
+			}
+
+			// ignore role-bindings not created by fission
+			if roleBinding.Name != fission.PackageGetterRB && roleBinding.Name != fission.SecretConfigMapGetterRB {
+				continue
+			}
+
+			// in order to find out if there are any functions that need this role-binding in role-binding namespace,
+			// we can list the functions once per role-binding.
+			funcList, err := fissionClient.Functions(roleBinding.Namespace).List(meta_v1.ListOptions{})
+			if err != nil {
+				log.Printf("Error fetching environment list in : %s", roleBinding.Namespace)
+				continue
+			}
+
+			// final map of service accounts that can be removed from this roleBinding object
+			// using a map here instead of a list so the code in RemoveSAFromRoleBindingWithRetries is efficient.
+			saToRemove := make(map[string]bool)
+
+			// the following flags are needed to decide if any of the service accounts can be removed from role-bindings depending on the functions that need them.
+			// ndmFunc denotes if there's at least one function that has executor type New deploy Manager
+			// funcEnvReference denotes if there's at least one function that has reference to an environment in the SA Namespace for the SA in question
+			var ndmFunc, funcEnvReference bool
+
+			// iterate through each subject in the role-binding and check if there are any references to them
+			for _, subj := range roleBinding.Subjects {
+				ndmFunc = false
+				funcEnvReference = false
+
+				// this is the reverse of what we're doing in setting up of role-bindings. if objects are created in default ns,
+				// the SA namespace will have the value of "fission-function"/"fission-builder" depending on the SA.
+				// so now we need to look for the objects in default namespace.
+				saNs := subj.Namespace
+				if subj.Namespace == functionNs ||
+					subj.Namespace == envBuilderNs {
+					saNs = meta_v1.NamespaceDefault
+				}
+
+				// go through each function and find out if there's either at least one function with env reference in the same namespace as the Service Account in this iteration
+				// or at least one function using ndm executor in the role-binding namespace and set the corresponding flags
+				for _, fn := range funcList.Items {
+					if fn.Spec.Environment.Namespace == saNs {
+						funcEnvReference = true
+						break
+					}
+
+					if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy {
+						ndmFunc = true
+						break
+					}
+				}
+
+				// if its a package-getterr-rb, we have 2 kinds of SAs and each of them is handled differently
+				// else if its a secret-configmap-rb, we have only one SA which is fission-fetcher
+				if roleBinding.Name == fission.PackageGetterRB {
+					// check if there is an env obj in saNs
+					envList, err := fissionClient.Environments(saNs).List(meta_v1.ListOptions{})
+					if err != nil {
+						log.Printf("Error fetching environment list in : %s", saNs)
+						continue
+					}
+
+					// if the SA in this iteration is fission-builder, then we need to only check
+					// if either there's at least one env object in the SA's namespace, or,
+					// if there's at least one function in the role-binding namespace with env reference
+					// to the SA's namespace.
+					// if neither, then we can remove this SA from this role-binding
+					if subj.Name == fission.FissionBuilderSA {
+						if len(envList.Items) == 0 && !funcEnvReference {
+							saToRemove[fission.MakeSAMapKey(subj.Name, subj.Namespace)] = true
+						}
+					}
+
+					// if the SA in this iteration is fission-fetcher, then in addition to above checks,
+					// we also need to check if there's at least one function with executor type New deploy
+					// in the rolebinding's namespace.
+					// if none of them are true, then remove this SA from this role-binding
+					if subj.Name == fission.FissionFetcherSA {
+						if len(envList.Items) == 0 && !ndmFunc && !funcEnvReference {
+							// remove SA from rolebinding
+							saToRemove[fission.MakeSAMapKey(subj.Name, subj.Namespace)] = true
+						}
+					}
+				} else if roleBinding.Name == fission.SecretConfigMapGetterRB {
+					// if there's not even one function in the role-binding's namespace and there's not even
+					// one function with env reference to the SA's namespace, then remove that SA
+					// from this role-binding
+					if !ndmFunc && !funcEnvReference {
+						saToRemove[fission.MakeSAMapKey(subj.Name, subj.Namespace)] = true
+					}
+				}
+			}
+
+			// finally, make a call to RemoveSAFromRoleBindingWithRetries for all the service accounts that need to be removed
+			// for the role-binding in this iteration
+			if len(saToRemove) != 0 {
+				log.Printf("saToRemove : %v for rolebinding : %s.%s", saToRemove, roleBinding.Name, roleBinding.Namespace)
+
+				// call this once in the end for each role-binding
+				err = fission.RemoveSAFromRoleBindingWithRetries(client, roleBinding.Name, roleBinding.Namespace, saToRemove)
+				if err != nil {
+					// if there's an error, we just log it and proceed with the next role-binding, hoping that this role-binding
+					// will be processed in next iteration.
+					log.Printf("Error removing SA : %v from rolebinding : %s.%s, err: %v", saToRemove, roleBinding.Name,
+						roleBinding.Namespace, err)
+				}
+			}
+		}
+
+		// some sleep before the next cleanup iteration
+		time.Sleep(cleanupRoleBindingInterval)
+	}
+}

executor/executor.go

@@ -70,6 +70,7 @@ func MakeExecutor(gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, fis
 		fsCreateWg:  make(map[string]*sync.WaitGroup),
 	}
 	go executor.serveCreateFuncServices()
+
 	return executor
 }
 
@@ -127,7 +128,6 @@ func (executor *Executor) getFunctionExecutorType(meta *metav1.ObjectMeta) (fiss
 		return "", err
 	}
 	return fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType, nil
-
 }
 
 func (executor *Executor) createServiceForFunction(meta *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
@@ -211,9 +211,9 @@ func serveMetric() {
 	log.Fatal(http.ListenAndServe(metricAddr, nil))
 }
 
-// StartExecutor Starts executor and the backend components that executor uses such as Poolmgr,
-// deploymgr and potential future backends
-func StartExecutor(fissionNamespace string, functionNamespace string, port int) error {
+// StartExecutor Starts executor and the executor components such as Poolmgr,
+// deploymgr and potential future executor types
+func StartExecutor(fissionNamespace string, functionNamespace string, envBuilderNamespace string, port int) error {
 	// setup a signal handler for SIGTERM
 	fission.SetupStackTraceHandler()
 
@@ -235,6 +235,7 @@ func StartExecutor(fissionNamespace string, functionNamespace string, port int)
 	poolID := strings.ToLower(uniuri.NewLen(8))
 	cleanupObjects(kubernetesClient, functionNamespace, poolID)
 	go idleObjectReaper(kubernetesClient, fissionClient, fsCache, time.Minute*2)
+	go cleanupRoleBindings(kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
 
 	gpm := poolmgr.MakeGenericPoolManager(
 		fissionClient, kubernetesClient,

executor/newdeploy/newdeploy.go

@@ -50,14 +50,14 @@ const (
 )
 
 func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Environment,
-	deployName string, deployLabels map[string]string) (*v1beta1.Deployment, error) {
+	deployName string, deployLabels map[string]string, deployNamespace string) (*v1beta1.Deployment, error) {
 
 	replicas := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
 	if replicas == 0 {
 		replicas = 1
 	}
 
-	existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deploy.namespace).Get(deployName, metav1.GetOptions{})
+	existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Get(deployName, metav1.GetOptions{})
 	if err == nil {
 		if existingDepl.Status.ReadyReplicas < replicas {
 			existingDepl, err = deploy.waitForDeploy(existingDepl, replicas)
@@ -66,13 +66,17 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro
 	}
 
 	if err != nil && k8s_err.IsNotFound(err) {
+		err := deploy.setupRBACObjs(deployNamespace, fn)
+		if err != nil {
+			return nil, err
+		}
 
 		deployment, err := deploy.getDeploymentSpec(fn, env, deployName, deployLabels)
 		if err != nil {
 			return nil, err
 		}
 
-		depl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deploy.namespace).Create(deployment)
+		depl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Create(deployment)
 		if err != nil {
 			log.Printf("Error while creating deployment: %v", err)
 			return nil, err
@@ -85,13 +89,39 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro
 
 }
 
-func (deploy *NewDeploy) getDeployment(fn *crd.Function) (*v1beta1.Deployment, error) {
-	deployName := deploy.getObjName(fn)
-	return deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deploy.namespace).Get(deployName, metav1.GetOptions{})
+func (deploy *NewDeploy) setupRBACObjs(deployNamespace string, fn *crd.Function) error {
+	// create fetcher SA in this ns, if not already created
+	_, err := fission.SetupSA(deploy.kubernetesClient, fission.FissionFetcherSA, deployNamespace)
+	if err != nil {
+		log.Printf("Error : %v creating %s in ns : %s for function: %s.%s", err, fission.FissionFetcherSA, deployNamespace, fn.Metadata.Name, fn.Metadata.Namespace)
+		return err
+	}
+
+	// create a cluster role binding for the fetcher SA, if not already created, granting access to do a get on packages in any ns
+	err = fission.SetupRoleBinding(deploy.kubernetesClient, fission.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace)
+	if err != nil {
+		log.Printf("Error : %v creating %s RoleBinding for function: %s.%s", err, fission.PackageGetterRB, fn.Metadata.Name, fn.Metadata.Namespace)
+		return err
+	}
+
+	// create rolebinding in function namespace for fetcherSA.envNamespace to be able to get secrets and configmaps
+	err = fission.SetupRoleBinding(deploy.kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace)
+	if err != nil {
+		log.Printf("Error : %v creating %s RoleBinding for function %s.%s", err, fission.SecretConfigMapGetterRB, fn.Metadata.Name, fn.Metadata.Namespace)
+		return err
+	}
+
+	log.Printf("Set up all RBAC objects for function : %s.%s", fn.Metadata.Name, fn.Metadata.Namespace)
+	return nil
 }
 
-func (deploy *NewDeploy) updateDeployment(deployment *v1beta1.Deployment) error {
-	_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deploy.namespace).Update(deployment)
+func (deploy *NewDeploy) getDeployment(fn *crd.Function) (*v1beta1.Deployment, error) {
+	deployName := deploy.getObjName(fn)
+	return deploy.kubernetesClient.ExtensionsV1beta1().Deployments(fn.Metadata.Namespace).Get(deployName, metav1.GetOptions{})
+}
+
+func (deploy *NewDeploy) updateDeployment(deployment *v1beta1.Deployment, ns string) error {
+	_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Update(deployment)
 	return err
 }
 
@@ -115,7 +145,9 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen
 	if replicas == 0 {
 		replicas = 1
 	}
+
 	targetFilename := "user"
+
 	gracePeriodSeconds := int64(6 * 60)
 	if env.Spec.TerminationGracePeriod > 0 {
 		gracePeriodSeconds = env.Spec.TerminationGracePeriod
@@ -351,7 +383,7 @@ func (deploy *NewDeploy) createOrGetHpa(hpaName string, execStrategy *fission.Ex
 	maxRepl := int32(execStrategy.MaxScale)
 	targetCPU := int32(execStrategy.TargetCPUPercent)
 
-	existingHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(deploy.namespace).Get(hpaName, metav1.GetOptions{})
+	existingHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Get(hpaName, metav1.GetOptions{})
 	if err == nil {
 		return existingHpa, err
 	}
@@ -364,7 +396,7 @@ func (deploy *NewDeploy) createOrGetHpa(hpaName string, execStrategy *fission.Ex
 		hpa := asv1.HorizontalPodAutoscaler{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      hpaName,
-				Namespace: deploy.namespace,
+				Namespace: depl.ObjectMeta.Namespace,
 				Labels:    depl.Labels,
 			},
 			Spec: asv1.HorizontalPodAutoscalerSpec{
@@ -379,7 +411,7 @@ func (deploy *NewDeploy) createOrGetHpa(hpaName string, execStrategy *fission.Ex
 			},
 		}
 
-		cHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(deploy.namespace).Create(&hpa)
+		cHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Create(&hpa)
 		if err != nil {
 			return nil, err
 		}
@@ -392,11 +424,11 @@ func (deploy *NewDeploy) createOrGetHpa(hpaName string, execStrategy *fission.Ex
 
 func (deploy *NewDeploy) getHpa(fn *crd.Function) (*asv1.HorizontalPodAutoscaler, error) {
 	hpaName := deploy.getObjName(fn)
-	return deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(deploy.namespace).Get(hpaName, metav1.GetOptions{})
+	return deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(fn.Metadata.Namespace).Get(hpaName, metav1.GetOptions{})
 }
 
 func (deploy *NewDeploy) updateHpa(hpa *asv1.HorizontalPodAutoscaler) error {
-	_, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(deploy.namespace).Update(hpa)
+	_, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Update(hpa)
 	return err
 }
 
@@ -405,9 +437,9 @@ func (deploy *NewDeploy) deleteHpa(ns string, name string) error {
 	return err
 }
 
-func (deploy *NewDeploy) createOrGetSvc(deployLabels map[string]string, svcName string) (*apiv1.Service, error) {
+func (deploy *NewDeploy) createOrGetSvc(deployLabels map[string]string, svcName string, svcNamespace string) (*apiv1.Service, error) {
 
-	existingSvc, err := deploy.kubernetesClient.CoreV1().Services(deploy.namespace).Get(svcName, metav1.GetOptions{})
+	existingSvc, err := deploy.kubernetesClient.CoreV1().Services(svcNamespace).Get(svcName, metav1.GetOptions{})
 	if err == nil {
 		return existingSvc, err
 	}
@@ -436,7 +468,7 @@ func (deploy *NewDeploy) createOrGetSvc(deployLabels map[string]string, svcName
 			},
 		}
 
-		svc, err := deploy.kubernetesClient.CoreV1().Services(deploy.namespace).Create(service)
+		svc, err := deploy.kubernetesClient.CoreV1().Services(svcNamespace).Create(service)
 		if err != nil {
 			return nil, err
 		}
@@ -457,7 +489,7 @@ func (deploy *NewDeploy) deleteSvc(ns string, name string) error {
 
 func (deploy *NewDeploy) waitForDeploy(depl *v1beta1.Deployment, replicas int32) (*v1beta1.Deployment, error) {
 	for i := 0; i < 120; i++ {
-		latestDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deploy.namespace).Get(depl.Name, metav1.GetOptions{})
+		latestDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(depl.ObjectMeta.Namespace).Get(depl.Name, metav1.GetOptions{})
 		if err != nil {
 			return nil, err
 		}

executor/newdeploy/newdeploymgr.go

@@ -145,7 +145,7 @@ func (deploy *NewDeploy) Run(ctx context.Context) {
 
 func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controller) {
 	resyncPeriod := 30 * time.Second
-	listWatch := k8sCache.NewListWatchFromClient(deploy.crdClient, "functions", metav1.NamespaceDefault, fields.Everything())
+	listWatch := k8sCache.NewListWatchFromClient(deploy.crdClient, "functions", metav1.NamespaceAll, fields.Everything())
 	store, controller := k8sCache.NewInformer(listWatch, &crd.Function{}, resyncPeriod, k8sCache.ResourceEventHandlerFuncs{
 		AddFunc: func(obj interface{}) {
 			fn := obj.(*crd.Function)
@@ -287,19 +287,26 @@ func (deploy *NewDeploy) fnCreate(fn *crd.Function) (*fscache.FuncSvc, error) {
 
 	deployLabels := deploy.getDeployLabels(fn, env)
 
+	// to support backward compatibility, if the function was created in default ns, we fall back to creating the
+	// deployment of the function in fission-function ns
+	ns := deploy.namespace
+	if fn.Metadata.Namespace != metav1.NamespaceDefault {
+		ns = fn.Metadata.Namespace
+	}
+
 	// Envoy(istio-proxy) returns 404 directly before istio pilot
 	// propagates latest Envoy-specific configuration.
 	// Since newdeploy waits for pods of deployment to be ready,
 	// change the order of kubeObject creation (create service first,
 	// then deployment) to take advantage of waiting time.
-	svc, err := deploy.createOrGetSvc(deployLabels, objName)
+	svc, err := deploy.createOrGetSvc(deployLabels, objName, ns)
 	if err != nil {
 		log.Printf("Error creating the service %v: %v", objName, err)
 		return fsvc, err
 	}
 	svcAddress := fmt.Sprintf("%v.%v", svc.Name, svc.Namespace)
 
-	depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels)
+	depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels, ns)
 	if err != nil {
 		log.Printf("Error creating the deployment %v: %v", objName, err)
 		return fsvc, err
@@ -468,7 +475,15 @@ func (deploy *NewDeploy) fnUpdate(oldFn *crd.Function, newFn *crd.Function) {
 			updateStatus(oldFn, err, "failed to get new deployment spec while updating function")
 			return
 		}
-		err = deploy.updateDeployment(newDeployment)
+
+		// to support backward compatibility, if the function was created in default ns, we fall back to creating the
+		// deployment of the function in fission-function ns
+		ns := deploy.namespace
+		if newFn.Metadata.Namespace != metav1.NamespaceDefault {
+			ns = newFn.Metadata.Namespace
+		}
+
+		err = deploy.updateDeployment(newDeployment, ns)
 		if err != nil {
 			updateStatus(oldFn, err, "failed to update deployment while updating function")
 			return
@@ -499,19 +514,26 @@ func (deploy *NewDeploy) fnDelete(fn *crd.Function) (*fscache.FuncSvc, error) {
 	}
 	objName := fsvc.Name
 
-	err = deploy.deleteDeployment(deploy.namespace, objName)
+	// to support backward compatibility, if the function was created in default ns, we fall back to creating the
+	// deployment of the function in fission-function ns, so cleaning up resources there
+	ns := deploy.namespace
+	if fn.Metadata.Namespace != metav1.NamespaceDefault {
+		ns = fn.Metadata.Namespace
+	}
+
+	err = deploy.deleteDeployment(ns, objName)
 	if err != nil {
 		log.Printf("Error deleting the deployment: %v", objName)
 		delError = err
 	}
 
-	err = deploy.deleteSvc(deploy.namespace, objName)
+	err = deploy.deleteSvc(ns, objName)
 	if err != nil {
 		log.Printf("Error deleting the service: %v", objName)
 		delError = err
 	}
 
-	err = deploy.deleteHpa(deploy.namespace, objName)
+	err = deploy.deleteHpa(ns, objName)
 	if err != nil {
 		log.Printf("Error deleting the HPA: %v", objName)
 		delError = err
@@ -520,6 +542,7 @@ func (deploy *NewDeploy) fnDelete(fn *crd.Function) (*fscache.FuncSvc, error) {
 	if delError != nil {
 		return nil, delError
 	}
+
 	return nil, nil
 }
 

executor/poolmgr/funcwatcher.go

@@ -26,7 +26,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
 	apiv1 "k8s.io/client-go/pkg/api/v1"
-	"k8s.io/client-go/rest"
 	k8sCache "k8s.io/client-go/tools/cache"
 
 	"github.com/fission/fission"
@@ -39,15 +38,15 @@ func getIstioServiceLabels(fnName string) map[string]string {
 	}
 }
 
-func makeFuncIstioServiceRegister(crdClient *rest.RESTClient,
-	kubernetesClient *kubernetes.Clientset, fnNamespace string) k8sCache.Controller {
+func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset, fissionfnNamespace string, istioEnabled bool) (k8sCache.Store, k8sCache.Controller) {
 
 	resyncPeriod := 30 * time.Second
-	lw := k8sCache.NewListWatchFromClient(crdClient, "functions", metav1.NamespaceDefault, fields.Everything())
-	_, controller := k8sCache.NewInformer(lw, &crd.Function{}, resyncPeriod,
+	lw := k8sCache.NewListWatchFromClient(fissionClient.GetCrdClient(), "functions", metav1.NamespaceAll, fields.Everything())
+
+	funcStore, controller := k8sCache.NewInformer(lw, &crd.Function{}, resyncPeriod,
 		k8sCache.ResourceEventHandlerFuncs{
 			AddFunc: func(obj interface{}) {
-
 				fn := obj.(*crd.Function)
 
 				// Since istio only allows accessing pod through k8s service,
@@ -56,70 +55,140 @@ func makeFuncIstioServiceRegister(crdClient *rest.RESTClient,
 				// Functions with executor type "Newdeploy" is specialized at
 				// pod starts. In this case, just ignore such functions.
 				fnExecutorType := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
-				if fnExecutorType != fission.ExecutorTypePoolmgr {
+
+				// In some cases, user may not enter the executorType explicitly, for example in his spec.yaml.
+				// we assume it to be of type poolmgr
+				if fnExecutorType != "" && fnExecutorType != fission.ExecutorTypePoolmgr {
 					return
 				}
 
-				// create a same name service for function
-				// since istio only allows the traffic to service
-				sel := map[string]string{
-					"functionName": fn.Metadata.Name,
-					"functionUid":  string(fn.Metadata.UID),
+				// create or update role-binding
+				envNs := fissionfnNamespace
+				if fn.Spec.Environment.Namespace != metav1.NamespaceDefault {
+					envNs = fn.Spec.Environment.Namespace
 				}
 
-				svcName := fission.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
+				// TODO : Just bring to your attention during review :
+				// setup rolebinding is tried, if it fails, we dont return. we just log an error and move on, because :
+				// 1. not all functions have secrets and/or configmaps, so things will work without this rolebinding in that case.
+				// 2. on the contrary, when the route is tried, the env fetcher logs will show a 403 forbidden message and same will be relayed to executor.
+				err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
+				if err != nil {
+					log.Printf("Error : %v creating %s RoleBinding", err, fission.SecretConfigMapGetterRB)
+				} else {
+					log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in func's ns for func : %s", fission.FissionFetcherSA, envNs, fn.Metadata.Name, fn.Metadata.Namespace)
+				}
 
-				// service for accepting user traffic
-				svc := apiv1.Service{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace: fnNamespace,
-						Name:      svcName,
-						Labels:    getIstioServiceLabels(fn.Metadata.Name),
-					},
-					Spec: apiv1.ServiceSpec{
-						Type: apiv1.ServiceTypeClusterIP,
-						Ports: []apiv1.ServicePort{
-							// Service port name should begin with a recognized prefix, or the traffic will be
-							// treated as TCP traffic. (https://istio.io/docs/setup/kubernetes/sidecar-injection.html)
-							// Originally the ports' name are similar to "http-fetch" and "http-specialize".
-							// But for istio 0.5.1, istio-proxy return unexpected 431 error with such naming.
-							// https://github.com/istio/istio/issues/928
-							// Workaround: remove prefix
-							// TODO: prepend prefix once the bug fixed
-							{
-								Name:       "fetch",
-								Protocol:   apiv1.ProtocolTCP,
-								Port:       8000,
-								TargetPort: intstr.FromInt(8000),
-							},
-							{
-								Name:       "specialize",
-								Protocol:   apiv1.ProtocolTCP,
-								Port:       8888,
-								TargetPort: intstr.FromInt(8888),
-							},
+				if istioEnabled {
+					// create a same name service for function
+					// since istio only allows the traffic to service
+					sel := map[string]string{
+						"functionName": fn.Metadata.Name,
+						"functionUid":  string(fn.Metadata.UID),
+					}
+
+					svcName := fission.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
+
+					// service for accepting user traffic
+					svc := apiv1.Service{
+						ObjectMeta: metav1.ObjectMeta{
+							Namespace: envNs,
+							Name:      svcName,
+							Labels:    getIstioServiceLabels(fn.Metadata.Name),
 						},
-						Selector: sel,
-					},
-				}
+						Spec: apiv1.ServiceSpec{
+							Type: apiv1.ServiceTypeClusterIP,
+							Ports: []apiv1.ServicePort{
+								// Service port name should begin with a recognized prefix, or the traffic will be
+								// treated as TCP traffic. (https://istio.io/docs/setup/kubernetes/sidecar-injection.html)
+								// Originally the ports' name are similar to "http-fetch" and "http-specialize".
+								// But for istio 0.5.1, istio-proxy return unexpected 431 error with such naming.
+								// https://github.com/istio/istio/issues/928
+								// Workaround: remove prefix
+								// TODO: prepend prefix once the bug fixed
+								{
+									Name:       "fetch",
+									Protocol:   apiv1.ProtocolTCP,
+									Port:       8000,
+									TargetPort: intstr.FromInt(8000),
+								},
+								{
+									Name:       "specialize",
+									Protocol:   apiv1.ProtocolTCP,
+									Port:       8888,
+									TargetPort: intstr.FromInt(8888),
+								},
+							},
+							Selector: sel,
+						},
+					}
 
-				// create function istio service if it does not exist
-				_, err := kubernetesClient.CoreV1().Services(fnNamespace).Create(&svc)
-				if err != nil && !kerrors.IsAlreadyExists(err) {
-					log.Printf("Error creating function istio service: %v", err)
+					// create function istio service if it does not exist
+					_, err = kubernetesClient.CoreV1().Services(envNs).Create(&svc)
+					if err != nil && !kerrors.IsAlreadyExists(err) {
+						log.Printf("Error creating function istio service: %v", err)
+					}
 				}
 			},
+
 			DeleteFunc: func(obj interface{}) {
 				fn := obj.(*crd.Function)
-				svcName := fission.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
-				// delete function istio service
-				err := kubernetesClient.CoreV1().Services(fnNamespace).Delete(svcName, nil)
-				if err != nil && !kerrors.IsNotFound(err) {
-					log.Printf("Error deleting function istio service: %v", err)
+
+				fnExecutorType := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
+				if fnExecutorType != "" && fnExecutorType != fission.ExecutorTypePoolmgr {
+					return
+				}
+
+				envNs := fissionfnNamespace
+				if fn.Spec.Environment.Namespace != metav1.NamespaceDefault {
+					envNs = fn.Spec.Environment.Namespace
+				}
+
+				if istioEnabled {
+					svcName := fission.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
+					// delete function istio service
+					err := kubernetesClient.CoreV1().Services(envNs).Delete(svcName, nil)
+					if err != nil && !kerrors.IsNotFound(err) {
+						log.Printf("Error deleting function istio service: %v", err)
+					}
+				}
+			},
+
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				oldFunc := oldObj.(*crd.Function)
+				newFunc := newObj.(*crd.Function)
+
+				if oldFunc.Metadata.ResourceVersion == newFunc.Metadata.ResourceVersion {
+					return
+				}
+
+				envChanged := (oldFunc.Spec.Environment.Namespace != newFunc.Spec.Environment.Namespace)
+
+				executorTypeChangedToPM := (oldFunc.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypePoolmgr &&
+					newFunc.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypePoolmgr)
+
+				// if a func's env reference gets updated and the newly referenced env is in a different ns,
+				// we need to create a rolebinding in func's ns so that the fetcher-sa in env ns has access
+				// to fetch secrets and config maps from the func's ns.
+				// similarly if executorType changed to Pool Manager, we now need a rolebinding in the func ns for fetcher sa
+				// present in env ns because for newdeploy, the fetcher sa is in function namespace
+				if envChanged || executorTypeChangedToPM {
+					envNs := fissionfnNamespace
+					if newFunc.Spec.Environment.Namespace != metav1.NamespaceDefault {
+						envNs = newFunc.Spec.Environment.Namespace
+					}
+
+					err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB,
+						newFunc.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole,
+						fission.FissionFetcherSA, envNs)
+					if err != nil {
+						log.Printf("Error : %v creating GetSecretConfigMapRoleBinding", err)
+					} else {
+						log.Printf("Set up rolebinding for fetcher SA in func's env ns : %s, in func's ns : %s, for func : %s", newFunc.Spec.Environment.Namespace, newFunc.Metadata.Namespace, newFunc.Metadata.Name)
+					}
 				}
 			},
-			UpdateFunc: func(oldObj, newObj interface{}) {},
 		})
 
-	return controller
+	return funcStore, controller
 }

executor/poolmgr/gp.go

@@ -56,6 +56,7 @@ type (
 		replicas               int32                         // num idle pods
 		deployment             *v1beta1.Deployment           // kubernetes deployment
 		namespace              string                        // namespace to keep our resources
+		functionNamespace      string                        // fallback namespace for fission functions
 		podReadyTimeout        time.Duration                 // timeout for generic pods to become ready
 		idlePodReapTime        time.Duration                 // pods unused for idlePodReapTime are deleted
 		fsCache                *fscache.FunctionServiceCache // cache funcSvc's by function, address and podname
@@ -103,6 +104,7 @@ func MakeGenericPool(
 	env *crd.Environment,
 	initialReplicas int32,
 	namespace string,
+	functionNamespace string,
 	fsCache *fscache.FunctionServiceCache,
 	instanceId string,
 	enableIstio bool) (*GenericPool, error) {
@@ -125,23 +127,24 @@ func MakeGenericPool(
 	// TODO: in general we need to provide the user a way to configure pools.  Initial
 	// replicas, autoscaling params, various timeouts, etc.
 	gp := &GenericPool{
-		env:              env,
-		replicas:         initialReplicas, // TODO make this an env param instead?
-		requestChannel:   make(chan *choosePodRequest),
-		fissionClient:    fissionClient,
-		kubernetesClient: kubernetesClient,
-		namespace:        namespace,
-		podReadyTimeout:  5 * time.Minute, // TODO make this an env param?
-		idlePodReapTime:  3 * time.Minute, // TODO make this configurable
-		fsCache:          fsCache,
-		poolInstanceId:   uniuri.NewLen(8),
-		instanceId:       instanceId,
-		fetcherImage:     fetcherImage,
-		useSvc:           false,       // defaults off -- svc takes a second or more to become routable, slowing cold start
-		useIstio:         enableIstio, // defaults off -- istio integration requires pod relabeling and it takes a second or more to become routable, slowing cold start
-		sharedMountPath:  "/userfunc", // change this may break v1 compatibility, since most of the v1 environments have hard-coded "/userfunc" in loading path
-		sharedSecretPath: "/secrets",
-		sharedCfgMapPath: "/configs",
+		env:               env,
+		replicas:          initialReplicas, // TODO make this an env param instead?
+		requestChannel:    make(chan *choosePodRequest),
+		fissionClient:     fissionClient,
+		kubernetesClient:  kubernetesClient,
+		namespace:         namespace,
+		functionNamespace: functionNamespace,
+		podReadyTimeout:   5 * time.Minute, // TODO make this an env param?
+		idlePodReapTime:   3 * time.Minute, // TODO make this configurable
+		fsCache:           fsCache,
+		poolInstanceId:    uniuri.NewLen(8),
+		instanceId:        instanceId,
+		fetcherImage:      fetcherImage,
+		useSvc:            false,       // defaults off -- svc takes a second or more to become routable, slowing cold start
+		useIstio:          enableIstio, // defaults off -- istio integration requires pod relabeling and it takes a second or more to become routable, slowing cold start
+		sharedMountPath:   "/userfunc", // change this may break v1 compatibility, since most of the v1 environments have hard-coded "/userfunc" in loading path
+		sharedSecretPath:  "/secrets",
+		sharedCfgMapPath:  "/configs",
 	}
 
 	gp.runtimeImagePullPolicy = getImagePullPolicy(runtimeImagePullPolicy)
@@ -149,6 +152,13 @@ func MakeGenericPool(
 	gp.fetcherImagePullPolicy = getImagePullPolicy(fetcherImagePullPolicy)
 	log.Printf("fetcher image: %v, pull policy: %v", gp.fetcherImage, gp.fetcherImagePullPolicy)
 
+	// create fetcher SA in this ns, if not already created
+	_, err := fission.SetupSA(gp.kubernetesClient, fission.FissionFetcherSA, gp.namespace)
+	if err != nil {
+		log.Printf("Error : %v creating fetcher SA in ns : %s", err, gp.namespace)
+		return nil, err
+	}
+
 	// Labels for generic deployment/RS/pods.
 	gp.labelsForPool = map[string]string{
 		"environmentName":                 gp.env.Metadata.Name,
@@ -158,7 +168,7 @@ func MakeGenericPool(
 	}
 
 	// create the pool
-	err := gp.createPool()
+	err = gp.createPool()
 	if err != nil {
 		return nil, err
 	}

executor/poolmgr/gpm.go

@@ -52,8 +52,11 @@ type (
 		instanceId     string
 		requestChannel chan *request
 
-		enableIstio          bool
-		istioServiceRegister k8sCache.Controller
+		enableIstio    bool
+		funcStore      k8sCache.Store
+		funcController k8sCache.Controller
+		pkgStore       k8sCache.Store
+		pkgController  k8sCache.Controller
 	}
 	request struct {
 		requestType
@@ -92,20 +95,19 @@ func MakeGenericPoolManager(
 			log.Println("Failed to parse ENABLE_ISTIO")
 		}
 		gpm.enableIstio = istio
-
-		if gpm.enableIstio {
-			gpm.istioServiceRegister = makeFuncIstioServiceRegister(
-				gpm.fissionClient.GetCrdClient(), gpm.kubernetesClient, functionNamespace)
-		}
 	}
 
+	gpm.funcStore, gpm.funcController = gpm.makeFuncController(
+		gpm.fissionClient, gpm.kubernetesClient, gpm.namespace, gpm.enableIstio)
+
+	gpm.pkgStore, gpm.pkgController = gpm.makePkgController(gpm.fissionClient, gpm.kubernetesClient, gpm.namespace)
+
 	return gpm
 }
 
 func (gpm *GenericPoolManager) Run(ctx context.Context) {
-	if gpm.enableIstio && gpm.istioServiceRegister != nil {
-		go gpm.istioServiceRegister.Run(ctx.Done())
-	}
+	go gpm.funcController.Run(ctx.Done())
+	go gpm.pkgController.Run(ctx.Done())
 }
 
 func (gpm *GenericPoolManager) service() {
@@ -113,6 +115,7 @@ func (gpm *GenericPoolManager) service() {
 		req := <-gpm.requestChannel
 		switch req.requestType {
 		case GET_POOL:
+			// just because they are missing in the cache, we end up creating another duplicate pool.
 			var err error
 			pool, ok := gpm.pools[crd.CacheKey(&req.env.Metadata)]
 			if !ok {
@@ -122,9 +125,16 @@ func (gpm *GenericPoolManager) service() {
 					poolsize = 1
 				}
 
+				// To support backward compatibility, if envs are created in default ns, we go ahead
+				// and create pools in fission-function ns as earlier.
+				ns := gpm.namespace
+				if req.env.Metadata.Namespace != metav1.NamespaceDefault {
+					ns = req.env.Metadata.Namespace
+				}
+
 				pool, err = MakeGenericPool(
 					gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize,
-					gpm.namespace, gpm.fsCache, gpm.instanceId, gpm.enableIstio)
+					ns, gpm.namespace, gpm.fsCache, gpm.instanceId, gpm.enableIstio)
 				if err != nil {
 					req.responseChannel <- &response{error: err}
 					continue
@@ -133,15 +143,12 @@ func (gpm *GenericPoolManager) service() {
 			}
 			req.responseChannel <- &response{pool: pool}
 		case CLEANUP_POOLS:
-			latestEnvSet := make(map[string]bool)
 			latestEnvPoolsize := make(map[string]int)
 			for _, env := range req.envList {
-				latestEnvSet[crd.CacheKey(&env.Metadata)] = true
 				latestEnvPoolsize[crd.CacheKey(&env.Metadata)] = int(gpm.getEnvPoolsize(&env))
 			}
 			for key, pool := range gpm.pools {
-				_, ok := latestEnvSet[key]
-				poolsize := latestEnvPoolsize[key]
+				poolsize, ok := latestEnvPoolsize[key]
 				if !ok || poolsize == 0 {
 					// Env no longer exists or pool size changed to zero
 

executor/poolmgr/packagewatcher.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package poolmgr
+
+import (
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/crd"
+)
+
+// TODO : It may make sense to make each of add, update, delete funcs run as separate go routines.
+func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset, fissionfnNamespace string) (k8sCache.Store, k8sCache.Controller) {
+
+	resyncPeriod := 30 * time.Second
+	lw := k8sCache.NewListWatchFromClient(fissionClient.GetCrdClient(), "packages", metav1.NamespaceAll, fields.Everything())
+	pkgStore, controller := k8sCache.NewInformer(lw, &crd.Package{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				pkg := obj.(*crd.Package)
+				log.Printf("List watch for package reported a new package addition: %s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)
+
+				// create or update role-binding for fetcher sa in env ns to be able to get the pkg contents from pkg namespace
+				envNs := fissionfnNamespace
+				if pkg.Spec.Environment.Namespace != metav1.NamespaceDefault {
+					envNs = pkg.Spec.Environment.Namespace
+				}
+
+				// here, we return if we hit an error during rolebinding setup. this is because this rolebinding is mandatory for
+				// every function's package to be loaded into its env. without that, there's no point to move forward.
+				err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
+				if err != nil {
+					log.Printf("Error creating %s for package: %s.%s, err: %v", fission.PackageGetterRB, pkg.Metadata.Name, pkg.Metadata.Namespace, err)
+					return
+				}
+
+				log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, pkg.Metadata.Namespace, pkg.Metadata.Name)
+			},
+
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				oldPkg := oldObj.(*crd.Package)
+				newPkg := newObj.(*crd.Package)
+
+				if oldPkg.Metadata.ResourceVersion == newPkg.Metadata.ResourceVersion {
+					return
+				}
+
+				// if a pkg's env reference gets updated and the newly referenced env is in a different ns,
+				// we need to update the role-binding in pkg ns to grant permissions to the fetcher-sa in env ns
+				// to do a get on pkg
+				if oldPkg.Spec.Environment.Namespace != newPkg.Spec.Environment.Namespace {
+					envNs := fissionfnNamespace
+					if newPkg.Spec.Environment.Namespace != metav1.NamespaceDefault {
+						envNs = newPkg.Spec.Environment.Namespace
+					}
+
+					err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB,
+						newPkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole,
+						fission.FissionFetcherSA, envNs)
+					if err != nil {
+						log.Printf("Error : %v updating %s for package: %s.%s", err, fission.PackageGetterRB, newPkg.Metadata.Name, newPkg.Metadata.Namespace)
+						return
+					}
+					log.Printf("Updated rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, newPkg.Metadata.Namespace, newPkg.Metadata.Name)
+				}
+			},
+		})
+
+	return pkgStore, controller
+}