Enabling multi-tenancy for fission objects. (#655)

This feature allows creation of fission objects in different namespaces, in addition to retaining the existing behavior of creating fission objects in default namespace if user doesnt provide one. 
It also removes cluster admin roles for fission-fetcher and fission-builder Service Accounts and grants them only those privileges that they need.

buildermgr/buildermgr.go

@@ -40,8 +40,8 @@ func Start(storageSvcUrl string, envBuilderNamespace string) error {
 	go envWatcher.watchEnvironments()
 
 	pkgWatcher := makePackageWatcher(fissionClient,
-		kubernetesClient.CoreV1().RESTClient(), envBuilderNamespace, storageSvcUrl)
-	go pkgWatcher.watchPackages()
+		kubernetesClient, envBuilderNamespace, storageSvcUrl)
+	go pkgWatcher.watchPackages(fissionClient, kubernetesClient, envBuilderNamespace)
 
 	select {}
 }

buildermgr/common.go

@@ -22,8 +22,6 @@ import (
 	"net/http"
 	"strings"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"github.com/dchest/uniuri"
 	"github.com/fission/fission"
 	"github.com/fission/fission/builder"
@@ -40,17 +38,17 @@ import (
 // 3. Send upload request to fetcher to upload deployment package.
 // 4. Return upload response and build logs.
 // *. Return build logs and error if any one of steps above failed.
-func buildPackage(fissionClient *crd.FissionClient, builderNamespace string,
+func buildPackage(fissionClient *crd.FissionClient, envBuilderNamespace string,
 	storageSvcUrl string, pkg *crd.Package) (uploadResp *fetcher.UploadResponse, buildLogs string, err error) {
 
-	env, err := fissionClient.Environments(metav1.NamespaceDefault).Get(pkg.Spec.Environment.Name)
+	env, err := fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
 	if err != nil {
 		e := fmt.Sprintf("Error getting environment CRD info: %v", err)
 		log.Println(e)
 		return nil, e, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
-	svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, builderNamespace)
+	svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, envBuilderNamespace)
 	srcPkgFilename := fmt.Sprintf("%v-%v", pkg.Metadata.Name, strings.ToLower(uniuri.NewLen(6)))
 	fetcherC := fetcherClient.MakeClient(fmt.Sprintf("http://%v:8000", svcName))
 	builderC := builderClient.MakeClient(fmt.Sprintf("http://%v:8001", svcName))
@@ -131,7 +129,7 @@ func updatePackage(fissionClient *crd.FissionClient,
 	}
 
 	// update package spec
-	pkg, err := fissionClient.Packages(metav1.NamespaceDefault).Update(pkg)
+	pkg, err := fissionClient.Packages(pkg.Metadata.Namespace).Update(pkg)
 	if err != nil {
 		log.Printf("Error updating package: %v", err)
 		return nil, err

buildermgr/envwatcher.go

@@ -42,7 +42,10 @@ const (
 	CLEANUP_BUILDERS
 
 	LABEL_ENV_NAME            = "envName"
+	LABEL_ENV_NAMESPACE       = "envNamespace"
 	LABEL_ENV_RESOURCEVERSION = "envResourceVersion"
+	LABEL_DEPLOYMENT_OWNER    = "owner"
+	BUILDER_MGR               = "buildermgr"
 )
 
 var (
@@ -130,14 +133,22 @@ func makeEnvironmentWatcher(fissionClient *crd.FissionClient,
 	return envWatcher
 }
 
-func (envw *environmentWatcher) getCacheKey(envName string, envResourceVersion string) string {
-	return fmt.Sprintf("%v-%v", envName, envResourceVersion)
+func (envw *environmentWatcher) getCacheKey(envName string, envNamespace string, envResourceVersion string) string {
+	return fmt.Sprintf("%v-%v-%v", envName, envNamespace, envResourceVersion)
 }
 
-func (envw *environmentWatcher) getLabels(envName string, envResourceVersion string) map[string]string {
+func (env *environmentWatcher) getLabelForDeploymentOwner() map[string]string {
+	return map[string]string{
+		LABEL_DEPLOYMENT_OWNER: BUILDER_MGR,
+	}
+}
+
+func (envw *environmentWatcher) getLabels(envName string, envNamespace string, envResourceVersion string) map[string]string {
 	return map[string]string{
 		LABEL_ENV_NAME:            envName,
+		LABEL_ENV_NAMESPACE:       envNamespace,
 		LABEL_ENV_RESOURCEVERSION: envResourceVersion,
+		LABEL_DEPLOYMENT_OWNER:    BUILDER_MGR,
 	}
 }
 
@@ -213,10 +224,17 @@ func (envw *environmentWatcher) service() {
 		req := <-envw.requestChan
 		switch req.requestType {
 		case GET_BUILDER:
-			key := envw.getCacheKey(req.env.Metadata.Name, req.env.Metadata.ResourceVersion)
+			// In order to support backward compatibility, for all environments with builder image created in default env,
+			// the pods will be created in fission-builder namespace
+			ns := envw.builderNamespace
+			if req.env.Metadata.Namespace != metav1.NamespaceDefault {
+				ns = req.env.Metadata.Namespace
+			}
+
+			key := envw.getCacheKey(req.env.Metadata.Name, ns, req.env.Metadata.ResourceVersion)
 			builderInfo, ok := envw.cache[key]
 			if !ok {
-				builderInfo, err := envw.createBuilder(req.env)
+				builderInfo, err := envw.createBuilder(req.env, ns)
 				if err != nil {
 					req.respChan <- envwResponse{err: err}
 					continue
@@ -229,7 +247,13 @@ func (envw *environmentWatcher) service() {
 			latestEnvList := make(map[string]*crd.Environment)
 			for i := range req.envList {
 				env := req.envList[i]
-				key := envw.getCacheKey(env.Metadata.Name, env.Metadata.ResourceVersion)
+				// In order to support backward compatibility, for all builder images created in default
+				// env, the pods are created in fission-builder namespace
+				ns := envw.builderNamespace
+				if env.Metadata.Namespace != metav1.NamespaceDefault {
+					ns = env.Metadata.Namespace
+				}
+				key := envw.getCacheKey(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
 				latestEnvList[key] = &env
 			}
 
@@ -239,38 +263,40 @@ func (envw *environmentWatcher) service() {
 			// cache and CRD. We need to iterate over the services &
 			// deployments to remove both normal and orphan builders.
 
-			svcList, err := envw.getBuilderServiceList(nil)
+			svcList, err := envw.getBuilderServiceList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
 			if err != nil {
 				log.Println(err.Error())
 			}
 			for _, svc := range svcList {
 				envName := svc.ObjectMeta.Labels[LABEL_ENV_NAME]
+				envNamespace := svc.ObjectMeta.Labels[LABEL_ENV_NAMESPACE]
 				envResourceVersion := svc.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION]
-				key := envw.getCacheKey(envName, envResourceVersion)
+				key := envw.getCacheKey(envName, envNamespace, envResourceVersion)
 				if _, ok := latestEnvList[key]; !ok {
-					err := envw.deleteBuilderService(svc.ObjectMeta.Labels)
+					err := envw.deleteBuilderServiceByName(svc.ObjectMeta.Name, svc.ObjectMeta.Namespace)
 					if err != nil {
 						log.Printf("Error removing builder service: %v", err)
 					}
 				}
-				delete(envw.cache, svc.ObjectMeta.Name)
+				delete(envw.cache, key)
 			}
 
-			deployList, err := envw.getBuilderDeploymentList(nil)
+			deployList, err := envw.getBuilderDeploymentList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
 			if err != nil {
 				log.Printf(err.Error())
 			}
 			for _, deploy := range deployList {
 				envName := deploy.ObjectMeta.Labels[LABEL_ENV_NAME]
+				envNamespace := deploy.ObjectMeta.Labels[LABEL_ENV_NAMESPACE]
 				envResourceVersion := deploy.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION]
-				key := envw.getCacheKey(envName, envResourceVersion)
+				key := envw.getCacheKey(envName, envNamespace, envResourceVersion)
 				if _, ok := latestEnvList[key]; !ok {
-					err := envw.deleteBuilderDeployment(deploy.ObjectMeta.Labels)
+					err := envw.deleteBuilderDeploymentByName(deploy.ObjectMeta.Name, deploy.ObjectMeta.Namespace)
 					if err != nil {
 						log.Printf("Error removing builder deployment: %v", err)
 					}
 				}
-				delete(envw.cache, deploy.ObjectMeta.Name)
+				delete(envw.cache, key)
 			}
 		}
 	}
@@ -294,19 +320,19 @@ func (envw *environmentWatcher) cleanupEnvBuilders(envs []crd.Environment) {
 	}
 }
 
-func (envw *environmentWatcher) createBuilder(env *crd.Environment) (*builderInfo, error) {
+func (envw *environmentWatcher) createBuilder(env *crd.Environment, ns string) (*builderInfo, error) {
 	var svc *apiv1.Service
 	var deploy *v1beta1.Deployment
 
-	sel := envw.getLabels(env.Metadata.Name, env.Metadata.ResourceVersion)
+	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
 
-	svcList, err := envw.getBuilderServiceList(sel)
+	svcList, err := envw.getBuilderServiceList(sel, ns)
 	if err != nil {
 		return nil, err
 	}
 	// there should be only one service in svcList
 	if len(svcList) == 0 {
-		svc, err = envw.createBuilderService(env)
+		svc, err = envw.createBuilderService(env, ns)
 		if err != nil {
 			return nil, fmt.Errorf("Error creating builder service: %v", err)
 		}
@@ -316,13 +342,20 @@ func (envw *environmentWatcher) createBuilder(env *crd.Environment) (*builderInf
 		return nil, fmt.Errorf("Found more than one builder service for environment %v", env.Metadata.Name)
 	}
 
-	deployList, err := envw.getBuilderDeploymentList(sel)
+	deployList, err := envw.getBuilderDeploymentList(sel, ns)
 	if err != nil {
 		return nil, err
 	}
 	// there should be only one deploy in deployList
 	if len(deployList) == 0 {
-		deploy, err = envw.createBuilderDeployment(env)
+		// create builder SA in this ns, if not already created
+		_, err := fission.SetupSA(envw.kubernetesClient, fission.FissionBuilderSA, ns)
+		if err != nil {
+			log.Printf("Error : %v creating %s in ns : %s", err, fission.FissionBuilderSA, ns)
+			return nil, err
+		}
+
+		deploy, err = envw.createBuilderDeployment(env, ns)
 		if err != nil {
 			return nil, fmt.Errorf("Error creating builder deployment: %v", err)
 		}
@@ -339,16 +372,44 @@ func (envw *environmentWatcher) createBuilder(env *crd.Environment) (*builderInf
 	}, nil
 }
 
-func (envw *environmentWatcher) deleteBuilderService(sel map[string]string) error {
-	svcList, err := envw.getBuilderServiceList(sel)
+func (envw *environmentWatcher) deleteBuilderServiceByName(name, namespace string) error {
+	falseVal := false
+	delOpt := &metav1.DeleteOptions{
+		OrphanDependents: &falseVal,
+	}
+	err := envw.kubernetesClient.
+		Services(namespace).
+		Delete(name, delOpt)
+	if err != nil {
+		return fmt.Errorf("Error deleting builder service %s.%s: %v", name, namespace, err)
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) deleteBuilderDeploymentByName(name, namespace string) error {
+	falseVal := false
+	delOpt := &metav1.DeleteOptions{
+		OrphanDependents: &falseVal,
+	}
+	err := envw.kubernetesClient.ExtensionsV1beta1().
+		Deployments(namespace).
+		Delete(name, delOpt)
+	if err != nil {
+		return fmt.Errorf("Error deleting builder deployment %s.%s: %v", name, namespace, err)
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) deleteBuilderService(sel map[string]string, ns string) error {
+	svcList, err := envw.getBuilderServiceList(sel, ns)
 	if err != nil {
 		return err
 	}
 	for _, svc := range svcList {
 		log.Printf("Removing builder service: %v", svc.ObjectMeta.Name)
 		err = envw.kubernetesClient.
-			Services(envw.builderNamespace).
-			Delete(svc.ObjectMeta.Name, nil)
+			Services(ns).
+			Delete(svc.ObjectMeta.Name, &delOpt)
 		if err != nil {
 			return fmt.Errorf("Error deleting builder service: %v", err)
 		}
@@ -356,15 +417,15 @@ func (envw *environmentWatcher) deleteBuilderService(sel map[string]string) erro
 	return nil
 }
 
-func (envw *environmentWatcher) deleteBuilderDeployment(sel map[string]string) error {
-	deployList, err := envw.getBuilderDeploymentList(sel)
+func (envw *environmentWatcher) deleteBuilderDeployment(sel map[string]string, ns string) error {
+	deployList, err := envw.getBuilderDeploymentList(sel, ns)
 	if err != nil {
 		return err
 	}
 	for _, deploy := range deployList {
 		log.Printf("Removing builder deployment: %v", deploy.ObjectMeta.Name)
 		err = envw.kubernetesClient.ExtensionsV1beta1().
-			Deployments(envw.builderNamespace).
+			Deployments(ns).
 			Delete(deploy.ObjectMeta.Name, &delOpt)
 		if err != nil {
 			return fmt.Errorf("Error deleteing builder deployment: %v", err)
@@ -373,8 +434,8 @@ func (envw *environmentWatcher) deleteBuilderDeployment(sel map[string]string) e
 	return nil
 }
 
-func (envw *environmentWatcher) getBuilderServiceList(sel map[string]string) ([]apiv1.Service, error) {
-	svcList, err := envw.kubernetesClient.Services(envw.builderNamespace).List(
+func (envw *environmentWatcher) getBuilderServiceList(sel map[string]string, ns string) ([]apiv1.Service, error) {
+	svcList, err := envw.kubernetesClient.Services(ns).List(
 		metav1.ListOptions{
 			LabelSelector: labels.Set(sel).AsSelector().String(),
 		})
@@ -384,12 +445,12 @@ func (envw *environmentWatcher) getBuilderServiceList(sel map[string]string) ([]
 	return svcList.Items, nil
 }
 
-func (envw *environmentWatcher) createBuilderService(env *crd.Environment) (*apiv1.Service, error) {
-	name := envw.getCacheKey(env.Metadata.Name, env.Metadata.ResourceVersion)
-	sel := envw.getLabels(env.Metadata.Name, env.Metadata.ResourceVersion)
+func (envw *environmentWatcher) createBuilderService(env *crd.Environment, ns string) (*apiv1.Service, error) {
+	name := fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion)
+	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
 	service := apiv1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: envw.builderNamespace,
+			Namespace: ns,
 			Name:      name,
 			Labels:    sel,
 		},
@@ -419,15 +480,15 @@ func (envw *environmentWatcher) createBuilderService(env *crd.Environment) (*api
 		},
 	}
 	log.Printf("Creating builder service: %v", name)
-	_, err := envw.kubernetesClient.Services(envw.builderNamespace).Create(&service)
+	_, err := envw.kubernetesClient.Services(ns).Create(&service)
 	if err != nil {
 		return nil, err
 	}
 	return &service, nil
 }
 
-func (envw *environmentWatcher) getBuilderDeploymentList(sel map[string]string) ([]v1beta1.Deployment, error) {
-	deployList, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(envw.builderNamespace).List(
+func (envw *environmentWatcher) getBuilderDeploymentList(sel map[string]string, ns string) ([]v1beta1.Deployment, error) {
+	deployList, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).List(
 		metav1.ListOptions{
 			LabelSelector: labels.Set(sel).AsSelector().String(),
 		})
@@ -437,12 +498,12 @@ func (envw *environmentWatcher) getBuilderDeploymentList(sel map[string]string)
 	return deployList.Items, nil
 }
 
-func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment) (*v1beta1.Deployment, error) {
+func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns string) (*v1beta1.Deployment, error) {
 	sharedMountPath := "/packages"
 	sharedCfgMapPath := "/configs"
 	sharedSecretPath := "/secrets"
-	name := envw.getCacheKey(env.Metadata.Name, env.Metadata.ResourceVersion)
-	sel := envw.getLabels(env.Metadata.Name, env.Metadata.ResourceVersion)
+	name := fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion)
+	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
 	var replicas int32 = 1
 
 	podAnnotation := make(map[string]string)
@@ -452,7 +513,7 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment) (*
 
 	deployment := &v1beta1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
-			Namespace: envw.builderNamespace,
+			Namespace: ns,
 			Name:      name,
 			Labels:    sel,
 		},
@@ -565,10 +626,11 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment) (*
 			},
 		},
 	}
-	log.Printf("Creating builder deployment: %v", envw.getCacheKey(env.Metadata.Name, env.Metadata.ResourceVersion))
-	_, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(envw.builderNamespace).Create(deployment)
+	log.Printf("Creating builder deployment: %v", fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion))
+	_, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Create(deployment)
 	if err != nil {
 		return nil, err
 	}
+
 	return deployment, nil
 }

buildermgr/pkgwatcher.go

@@ -30,26 +30,29 @@ import (
 	"github.com/fission/fission"
 	"github.com/fission/fission/cache"
 	"github.com/fission/fission/crd"
+	"k8s.io/client-go/kubernetes"
 )
 
 type (
 	packageWatcher struct {
 		fissionClient    *crd.FissionClient
+		k8sClient        *kubernetes.Clientset
 		podStore         k8sCache.Store
+		pkgStore         k8sCache.Store
 		builderNamespace string
 		storageSvcUrl    string
 	}
 )
 
-func makePackageWatcher(fissionClient *crd.FissionClient, getter k8sCache.Getter,
+func makePackageWatcher(fissionClient *crd.FissionClient, k8sClientSet *kubernetes.Clientset,
 	builderNamespace string, storageSvcUrl string) *packageWatcher {
-
-	lw := k8sCache.NewListWatchFromClient(getter, "pods", builderNamespace, fields.Everything())
+	lw := k8sCache.NewListWatchFromClient(k8sClientSet.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.Everything())
 	store, controller := k8sCache.NewInformer(lw, &apiv1.Pod{}, 30*time.Second, k8sCache.ResourceEventHandlerFuncs{})
 	go controller.Run(make(chan struct{}))
 
 	pkgw := &packageWatcher{
 		fissionClient:    fissionClient,
+		k8sClient:        k8sClientSet,
 		podStore:         store,
 		builderNamespace: builderNamespace,
 		storageSvcUrl:    storageSvcUrl,
@@ -118,8 +121,16 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
 		for _, item := range items {
 			pod := item.(*apiv1.Pod)
 
+			// In order to support backward compatibility, for all builder images created in default env,
+			// the pods will be created in fission-builder namespace
+			builderNs := pkgw.builderNamespace
+			if env.Metadata.Namespace != metav1.NamespaceDefault {
+				builderNs = env.Metadata.Namespace
+			}
+
 			// Filter non-matching pods
 			if pod.ObjectMeta.Labels[LABEL_ENV_NAME] != env.Metadata.Name ||
+				pod.ObjectMeta.Labels[LABEL_ENV_NAMESPACE] != builderNs ||
 				pod.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION] != env.Metadata.ResourceVersion {
 				continue
 			}
@@ -138,8 +149,18 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
 				break
 			}
 
-			uploadResp, buildLogs, err := buildPackage(pkgw.fissionClient,
-				pkgw.builderNamespace, pkgw.storageSvcUrl, pkg)
+			// Add the package getter rolebinding to builder sa
+			// we continue here if role binding was not setup succeesffully. this is because without this, the fetcher wont be able to fetch the source pkg into the container and
+			// the build will fail eventually
+			err := fission.SetupRoleBinding(pkgw.k8sClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, builderNs)
+			if err != nil {
+				log.Printf("Error : %v in setting up the role binding %s for pkg : %s.%s", err, fission.PackageGetterRB, pkg.Metadata.Name, pkg.Metadata.Namespace)
+				continue
+			} else {
+				log.Printf("Setup rolebinding for sa : %s.%s for pkg : %s.%s", fission.FissionBuilderSA, builderNs, pkg.Metadata.Name, pkg.Metadata.Namespace)
+			}
+
+			uploadResp, buildLogs, err := buildPackage(pkgw.fissionClient, builderNs, pkgw.storageSvcUrl, pkg)
 			if err != nil {
 				log.Printf("Error building package %v: %v", pkg.Metadata.Name, err)
 				updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
@@ -149,7 +170,7 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
 			log.Printf("Start updating info of package: %v", pkg.Metadata.Name)
 
 			fnList, err := pkgw.fissionClient.
-				Functions(metav1.NamespaceDefault).List(metav1.ListOptions{})
+				Functions(metav1.NamespaceAll).List(metav1.ListOptions{})
 			if err != nil {
 				e := fmt.Sprintf("Error getting function list: %v", err)
 				log.Println(e)
@@ -191,13 +212,18 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
 	// build timeout
 	updatePackage(pkgw.fissionClient, pkg,
 		fission.BuildStatusFailed, "Build timeout due to environment builder not ready", nil)
+
+	log.Printf("Max retries exceeded in building the source pkg : %s.%s, timeout due to environment builder not ready",
+		pkg.Metadata.Name, pkg.Metadata.Namespace)
+
 	return
 }
 
-func (pkgw *packageWatcher) watchPackages() {
+func (pkgw *packageWatcher) watchPackages(fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset, builderNamespace string) {
 	buildCache := cache.MakeCache(0, 0)
-	lw := k8sCache.NewListWatchFromClient(pkgw.fissionClient.GetCrdClient(), "packages", apiv1.NamespaceDefault, fields.Everything())
-	_, controller := k8sCache.NewInformer(lw, &crd.Package{}, 60*time.Second, k8sCache.ResourceEventHandlerFuncs{
+	lw := k8sCache.NewListWatchFromClient(pkgw.fissionClient.GetCrdClient(), "packages", apiv1.NamespaceAll, fields.Everything())
+	pkgStore, controller := k8sCache.NewInformer(lw, &crd.Package{}, 60*time.Second, k8sCache.ResourceEventHandlerFuncs{
 		AddFunc: func(obj interface{}) {
 			pkg := obj.(*crd.Package)
 			go pkgw.build(buildCache, pkg)
@@ -207,5 +233,6 @@ func (pkgw *packageWatcher) watchPackages() {
 			go pkgw.build(buildCache, pkg)
 		},
 	})
+	pkgw.pkgStore = pkgStore
 	controller.Run(make(chan struct{}))
 }