Block build requests until environment builder is ready (#437)

* Add readiness probe
* Remove builder manager http api interface since we don’t use/need it
* Check environment builder status and block build requests until builder is ready
* Replace deprecated api extension interface
* Add healthy check to python env

buildermgr/api.go

@@ -1,115 +0,0 @@
-/*
-Copyright 2017 The Fission Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package buildermgr
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-
-	"github.com/gorilla/handlers"
-	"github.com/gorilla/mux"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
-
-	"github.com/fission/fission"
-	"github.com/fission/fission/crd"
-)
-
-type (
-	BuildRequest struct {
-		Package metav1.ObjectMeta `json:"package"`
-	}
-
-	BuilderMgr struct {
-		fissionClient *crd.FissionClient
-		storageSvcUrl string
-		namespace     string
-	}
-)
-
-func MakeBuilderMgr(fissionClient *crd.FissionClient,
-	kubernetesClient *kubernetes.Clientset, storageSvcUrl string,
-	envBuilderNamespace string) *BuilderMgr {
-
-	envWatcher := makeEnvironmentWatcher(fissionClient, kubernetesClient, envBuilderNamespace)
-	go envWatcher.watchEnvironments()
-
-	pkgWatcher := makePackageWatcher(fissionClient, envBuilderNamespace, storageSvcUrl)
-	go pkgWatcher.watchPackages()
-
-	return &BuilderMgr{
-		fissionClient: fissionClient,
-		storageSvcUrl: storageSvcUrl,
-		namespace:     envBuilderNamespace,
-	}
-}
-
-func (builderMgr *BuilderMgr) build(w http.ResponseWriter, r *http.Request) {
-	body, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		e := fmt.Sprintf("Failed to read request: %v", err)
-		log.Println(e)
-		http.Error(w, e, 500)
-		return
-	}
-
-	buildReq := BuildRequest{}
-	err = json.Unmarshal([]byte(body), &buildReq)
-	if err != nil {
-		e := fmt.Sprintf("Invalid request body: %v", err)
-		log.Println(e)
-		http.Error(w, e, 400)
-		return
-	}
-
-	pkg, err := builderMgr.fissionClient.
-		Packages(buildReq.Package.Namespace).
-		Get(buildReq.Package.Name)
-	if err != nil {
-		e := fmt.Sprintf("Error getting package CRD info: %v", err)
-		log.Println(e)
-		http.Error(w, e, 500)
-		return
-	}
-
-	buildLogs, err := buildPackage(builderMgr.fissionClient, builderMgr.namespace, builderMgr.storageSvcUrl, pkg)
-	if err != nil {
-		code, e := fission.GetHTTPError(err)
-		http.Error(w, e, code)
-		return
-	}
-
-	w.Header().Set("Content-Type", "text/plain")
-	_, err = w.Write([]byte(buildLogs))
-	if err != nil {
-		e := fmt.Sprintf("Failed to reply http request: %v", err)
-		log.Println(e)
-		http.Error(w, e, 500)
-	}
-}
-
-func (builderMgr *BuilderMgr) Serve(port int) {
-	r := mux.NewRouter()
-	r.HandleFunc("/v1/build", builderMgr.build).Methods("POST")
-	address := fmt.Sprintf(":%v", port)
-	log.Printf("Start buildermgr at port %v", address)
-	log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r)))
-}

buildermgr/buildermgr.go

@@ -23,17 +23,18 @@ import (
 )
 
 // Start the buildermgr service.
-func Start(port int, storageSvcUrl string, envBuilderNamespace string) error {
+func Start(storageSvcUrl string, envBuilderNamespace string) error {
 	fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
 	if err != nil {
 		log.Printf("Failed to get kubernetes client: %v", err)
 		return err
 	}
 
-	api := MakeBuilderMgr(fissionClient, kubernetesClient,
-		storageSvcUrl, envBuilderNamespace)
+	envWatcher := makeEnvironmentWatcher(fissionClient, kubernetesClient, envBuilderNamespace)
+	go envWatcher.watchEnvironments()
 
-	go api.Serve(port)
+	pkgWatcher := makePackageWatcher(fissionClient, kubernetesClient.CoreV1().RESTClient(), envBuilderNamespace, storageSvcUrl)
+	go pkgWatcher.watchPackages()
 
-	return nil
+	select {}
 }

buildermgr/client/client.go

@@ -1,61 +0,0 @@
-/*
-Copyright 2017 The Fission Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package client
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"strings"
-
-	"github.com/fission/fission"
-	"github.com/fission/fission/buildermgr"
-)
-
-type (
-	Client struct {
-		url string
-	}
-)
-
-func MakeClient(builderUrl string) *Client {
-	return &Client{
-		url: strings.TrimSuffix(builderUrl, "/") + "/v1",
-	}
-}
-
-func (c *Client) PackageBuild(req *buildermgr.BuildRequest) ([]byte, error) {
-	body, err := json.Marshal(req)
-	if err != nil {
-		return nil, err
-	}
-	resp, err := http.Post(c.url+"/build", "application/json", bytes.NewReader(body))
-	if err != nil {
-		return nil, err
-	}
-	return c.handleResponse(resp)
-}
-
-func (c *Client) handleResponse(resp *http.Response) ([]byte, error) {
-	if resp.StatusCode != 200 {
-		return nil, fission.MakeErrorFromHTTP(resp)
-	}
-	defer resp.Body.Close()
-	body, err := ioutil.ReadAll(resp.Body)
-	return body, err
-}

buildermgr/common.go

@@ -34,48 +34,20 @@ import (
 )
 
 // buildPackage helps to build source package into deployment package.
-// Following is steps buildPackage takes to complete the build process.
-// 1. Check package status
-// 2. Update package status to running state
-// 3. Send fetch request to fetcher to fetch source package.
-// 4. Send build request to builder to start a build.
-// 5. Send upload request to fetcher to upload deployment package.
-// 6. Update package status to succeed state
-// 7. Update package resource in package ref of functions that share the same package
-// *. Update package status to failed state,if any one of steps above failed
+// Following is the steps buildPackage function takes to complete the whole process.
+// 1. Send fetch request to fetcher to fetch source package.
+// 2. Send build request to builder to start a build.
+// 3. Send upload request to fetcher to upload deployment package.
+// 4. Return upload response and build logs.
+// *. Return build logs and error if any one of steps above failed.
 func buildPackage(fissionClient *crd.FissionClient, builderNamespace string,
-	storageSvcUrl string, pkg *crd.Package) (buildLogs string, err error) {
-
-	// Only do build for pending packages
-	if pkg.Status.BuildStatus != fission.BuildStatusPending {
-		e := "package is not in pending state"
-		log.Println(e)
-		return e, fission.MakeError(http.StatusBadRequest, e)
-	}
-
-	// update package status to running state, so that
-	// we can know what status a package is through cli.
-	newPkgRV, err := updatePackage(fissionClient, pkg, fission.BuildStatusRunning, "", nil)
-
-	// Kubernetes checks resource version before applying
-	// new resource config. The update operation will be
-	// rejected if the resource version in metadata is lower
-	// than the latest version. Set resource version with
-	// latest return from updatePackage.
-	pkg.Metadata.ResourceVersion = newPkgRV
-	if err != nil {
-		e := fmt.Sprintf("Error setting package pending state: %v", err)
-		log.Println(e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
-		return e, fission.MakeError(http.StatusInternalServerError, e)
-	}
+	storageSvcUrl string, pkg *crd.Package) (uploadResp *fetcher.UploadResponse, buildLogs string, err error) {
 
 	env, err := fissionClient.Environments(metav1.NamespaceDefault).Get(pkg.Spec.Environment.Name)
 	if err != nil {
 		e := fmt.Sprintf("Error getting environment CRD info: %v", err)
 		log.Println(e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
-		return e, fission.MakeError(http.StatusInternalServerError, e)
+		return nil, e, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
 	svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, builderNamespace)
@@ -94,8 +66,7 @@ func buildPackage(fissionClient *crd.FissionClient, builderNamespace string,
 	if err != nil {
 		e := fmt.Sprintf("Error fetching source package: %v", err)
 		log.Println(e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
-		return e, fission.MakeError(http.StatusInternalServerError, e)
+		return nil, e, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
 	buildCmd := pkg.Spec.BuildCommand
@@ -119,8 +90,7 @@ func buildPackage(fissionClient *crd.FissionClient, builderNamespace string,
 			buildLogs = buildResp.BuildLogs
 		}
 		buildLogs += fmt.Sprintf("%v\n", e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
-		return e, fission.MakeError(http.StatusInternalServerError, e)
+		return nil, buildResp.BuildLogs, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
 	log.Printf("Build succeed, source package: %v, deployment package: %v", srcPkgFilename, buildResp.ArtifactFilename)
@@ -132,65 +102,20 @@ func buildPackage(fissionClient *crd.FissionClient, builderNamespace string,
 
 	log.Printf("Start uploading deployment package: %v", buildResp.ArtifactFilename)
 	// ask fetcher to upload the deployment package
-	uploadResp, err := fetcherC.Upload(uploadReq)
+	uploadResp, err = fetcherC.Upload(uploadReq)
 	if err != nil {
 		e := fmt.Sprintf("Error uploading deployment package: %v", err)
 		log.Println(e)
 		buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildResp.BuildLogs, nil)
-		return e, fission.MakeError(http.StatusInternalServerError, e)
+		return nil, buildResp.BuildLogs, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
-	log.Printf("Start updating info of package: %v", pkg.Metadata.Name)
-	// update package status and also build logs
-	newPkgRV, err = updatePackage(fissionClient, pkg,
-		fission.BuildStatusSucceeded, buildResp.BuildLogs, uploadResp)
-	pkg.Metadata.ResourceVersion = newPkgRV
-	if err != nil {
-		e := fmt.Sprintf("Error creating deployment package CRD resource: %v", err)
-		log.Println(e)
-		buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildResp.BuildLogs, nil)
-		return e, fission.MakeError(http.StatusInternalServerError, e)
-	}
-
-	fnList, err := fissionClient.
-		Functions(metav1.NamespaceDefault).List(metav1.ListOptions{})
-	if err != nil {
-		e := fmt.Sprintf("Error getting function list: %v", err)
-		log.Println(e)
-		buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildResp.BuildLogs, nil)
-		return e, fission.MakeError(http.StatusInternalServerError, e)
-	}
-
-	// A package may be used by multiple functions. Update
-	// functions with old package resource version
-	for _, fn := range fnList.Items {
-		if fn.Spec.Package.PackageRef.Name == pkg.Metadata.Name &&
-			fn.Spec.Package.PackageRef.Namespace == pkg.Metadata.Namespace &&
-			fn.Spec.Package.PackageRef.ResourceVersion != pkg.Metadata.ResourceVersion {
-			fn.Spec.Package.PackageRef.ResourceVersion = newPkgRV
-			// update CRD
-			_, err = fissionClient.Functions(fn.Metadata.Namespace).Update(&fn)
-			if err != nil {
-				e := fmt.Sprintf("Error updating function package resource version: %v", err)
-				log.Println(e)
-				buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
-				updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildResp.BuildLogs, nil)
-				return e, fission.MakeError(http.StatusInternalServerError, e)
-			}
-		}
-	}
-
-	log.Printf("Completed build request for package: %v", pkg.Metadata.Name)
-
-	return buildResp.BuildLogs, nil
+	return uploadResp, buildResp.BuildLogs, nil
 }
 
 func updatePackage(fissionClient *crd.FissionClient,
 	pkg *crd.Package, status fission.BuildStatus, buildLogs string,
-	uploadResp *fetcher.UploadResponse) (string, error) {
+	uploadResp *fetcher.UploadResponse) (*crd.Package, error) {
 
 	pkg.Status = fission.PackageStatus{
 		BuildStatus: status,
@@ -209,9 +134,9 @@ func updatePackage(fissionClient *crd.FissionClient,
 	pkg, err := fissionClient.Packages(metav1.NamespaceDefault).Update(pkg)
 	if err != nil {
 		log.Printf("Error updating package: %v", err)
-		return "", err
+		return nil, err
 	}
 
 	// return resource version for function to update function package ref
-	return pkg.Metadata.ResourceVersion, nil
+	return pkg, nil
 }

buildermgr/envwatcher.go

@@ -458,6 +458,19 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment) (*
 								},
 							},
 							Command: []string{"/builder", sharedMountPath},
+							ReadinessProbe: &apiv1.Probe{
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       2,
+								Handler: apiv1.Handler{
+									HTTPGet: &apiv1.HTTPGetAction{
+										Path: "/healthz",
+										Port: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: 8001,
+										},
+									},
+								},
+							},
 						},
 						{
 							Name:                   "fetcher",
@@ -471,6 +484,19 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment) (*
 								},
 							},
 							Command: []string{"/fetcher", sharedMountPath},
+							ReadinessProbe: &apiv1.Probe{
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       2,
+								Handler: apiv1.Handler{
+									HTTPGet: &apiv1.HTTPGetAction{
+										Path: "/healthz",
+										Port: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: 8000,
+										},
+									},
+								},
+							},
 						},
 					},
 					ServiceAccountName: "fission-builder",

buildermgr/pkgwatcher.go

@@ -17,68 +17,194 @@ limitations under the License.
 package buildermgr
 
 import (
+	"fmt"
 	"log"
 	"time"
 
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apimachinery/pkg/fields"
+	apiv1 "k8s.io/client-go/pkg/api/v1"
+	k8sCache "k8s.io/client-go/tools/cache"
 
 	"github.com/fission/fission"
+	"github.com/fission/fission/cache"
 	"github.com/fission/fission/crd"
 )
 
 type (
 	packageWatcher struct {
 		fissionClient    *crd.FissionClient
+		podStore         k8sCache.Store
 		builderNamespace string
 		storageSvcUrl    string
 	}
 )
 
-func makePackageWatcher(fissionClient *crd.FissionClient,
+func makePackageWatcher(fissionClient *crd.FissionClient, getter k8sCache.Getter,
 	builderNamespace string, storageSvcUrl string) *packageWatcher {
+
+	lw := k8sCache.NewListWatchFromClient(getter, "pods", builderNamespace, fields.Everything())
+	store, controller := k8sCache.NewInformer(lw, &apiv1.Pod{}, 30*time.Second, k8sCache.ResourceEventHandlerFuncs{})
+	go controller.Run(make(chan struct{}))
+
 	pkgw := &packageWatcher{
 		fissionClient:    fissionClient,
+		podStore:         store,
 		builderNamespace: builderNamespace,
 		storageSvcUrl:    storageSvcUrl,
 	}
 	return pkgw
 }
 
-func (pkgw *packageWatcher) build(pkg *crd.Package) {
-	_, err := buildPackage(pkgw.fissionClient, pkgw.builderNamespace, pkgw.storageSvcUrl, pkg)
-	if err != nil {
-		log.Printf("Error building package %v: %v", pkg.Metadata.Name, err)
+// build helps to update package status, checks environment builder pod status and
+// dispatches buildPackage to build source package into deployment package.
+// Following is the steps build function takes to complete the whole process.
+// 1. Check package status
+// 2. Update package status to running state
+// 3. Check environment builder pod status
+// 4. Call buildPackage to build package
+// 5. Update package resource in package ref of functions that share the same package
+// 6. Update package status to succeed state
+// *. Update package status to failed state,if any one of steps above failed/time out
+func (pkgw *packageWatcher) build(buildCache *cache.Cache, pkg *crd.Package) {
+
+	// Ignore non-pending state packages.
+	if pkg.Status.BuildStatus != fission.BuildStatusPending {
+		return
 	}
+
+	// Ignore duplicate build requests
+	key := fmt.Sprintf("%v-%v", pkg.Metadata.Name, pkg.Metadata.ResourceVersion)
+	err, _ := buildCache.Set(key, pkg)
+	if err != nil {
+		return
+	}
+	defer buildCache.Delete(key)
+
+	log.Printf("Start build for package %v with resource version %v", pkg.Metadata.Name, pkg.Metadata.ResourceVersion)
+
+	pkg, err = updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusRunning, "", nil)
+	if err != nil {
+		e := fmt.Sprintf("Error setting package pending state: %v", err)
+		log.Println(e)
+		updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		return
+	}
+
+	env, err := pkgw.fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
+	if errors.IsNotFound(err) {
+		updatePackage(pkgw.fissionClient, pkg,
+			fission.BuildStatusFailed, "Environment not existed", nil)
+		return
+	}
+
+	// Do health check for environment builder pod
+	for i := 0; i < 15; i++ {
+		// Informer store is not able to use label to find the pod,
+		// iterate all available environment builders.
+		items := pkgw.podStore.List()
+		if err != nil {
+			log.Printf("Error retrieving pod information for env %v: %v", err, env.Metadata.Name)
+			return
+		}
+
+		if len(items) == 0 {
+			log.Printf("Environment \"%v\" builder pod is not existed yet, retry again later.", pkg.Spec.Environment.Name)
+			time.Sleep(time.Duration(i*1) * time.Second)
+			continue
+		}
+
+		for _, item := range items {
+			pod := item.(*apiv1.Pod)
+
+			// Filter non-matching pods
+			if pod.ObjectMeta.Labels[LABEL_ENV_NAME] != env.Metadata.Name ||
+				pod.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION] != env.Metadata.ResourceVersion {
+				continue
+			}
+
+			// Pod may become "Running" state but still failed at health check, so use
+			// pod.Status.ContainerStatuses instead of pod.Status.Phase to check pod readiness states.
+			podIsReady := true
+
+			for _, cStatus := range pod.Status.ContainerStatuses {
+				podIsReady = podIsReady && cStatus.Ready
+			}
+
+			if !podIsReady {
+				log.Printf("Environment \"%v\" builder pod is not ready, retry again later.", pkg.Spec.Environment.Name)
+				time.Sleep(time.Duration(i*1) * time.Second)
+				break
+			}
+
+			uploadResp, buildLogs, err := buildPackage(pkgw.fissionClient, pkgw.builderNamespace, pkgw.storageSvcUrl, pkg)
+			if err != nil {
+				log.Printf("Error building package %v: %v", pkg.Metadata.Name, err)
+				updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
+				return
+			}
+
+			log.Printf("Start updating info of package: %v", pkg.Metadata.Name)
+
+			fnList, err := pkgw.fissionClient.
+				Functions(metav1.NamespaceDefault).List(metav1.ListOptions{})
+			if err != nil {
+				e := fmt.Sprintf("Error getting function list: %v", err)
+				log.Println(e)
+				buildLogs += fmt.Sprintf("%v\n", e)
+				updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
+			}
+
+			// A package may be used by multiple functions. Update
+			// functions with old package resource version
+			for _, fn := range fnList.Items {
+				if fn.Spec.Package.PackageRef.Name == pkg.Metadata.Name &&
+					fn.Spec.Package.PackageRef.Namespace == pkg.Metadata.Namespace &&
+					fn.Spec.Package.PackageRef.ResourceVersion != pkg.Metadata.ResourceVersion {
+					fn.Spec.Package.PackageRef.ResourceVersion = pkg.Metadata.ResourceVersion
+					// update CRD
+					_, err = pkgw.fissionClient.Functions(fn.Metadata.Namespace).Update(&fn)
+					if err != nil {
+						e := fmt.Sprintf("Error updating function package resource version: %v", err)
+						log.Println(e)
+						buildLogs += fmt.Sprintf("%v\n", e)
+						updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
+						return
+					}
+				}
+			}
+
+			_, err = updatePackage(pkgw.fissionClient, pkg,
+				fission.BuildStatusSucceeded, buildLogs, uploadResp)
+			if err != nil {
+				log.Printf("Error update package info: %v", err)
+				updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
+				return
+			}
+
+			log.Printf("Completed build request for package: %v", pkg.Metadata.Name)
+			return
+		}
+	}
+	// build timeout
+	updatePackage(pkgw.fissionClient, pkg,
+		fission.BuildStatusFailed, "Build timeout due to environment builder not ready", nil)
+	return
 }
 
 func (pkgw *packageWatcher) watchPackages() {
-	rv := ""
-	for {
-		wi, err := pkgw.fissionClient.Packages(metav1.NamespaceDefault).Watch(metav1.ListOptions{
-			ResourceVersion: rv,
-		})
-		if err != nil {
-			log.Fatalf("Error watching package CRD resources: %v", err)
-		}
-
-		for {
-			ev, more := <-wi.ResultChan()
-			if !more {
-				break
-			}
-			if ev.Type == watch.Error {
-				rv = ""
-				time.Sleep(time.Second)
-				break
-			}
-			pkg := ev.Object.(*crd.Package)
-			rv = pkg.Metadata.ResourceVersion
-
-			// only do build for packages in pending state
-			if pkg.Status.BuildStatus == fission.BuildStatusPending {
-				go pkgw.build(pkg)
-			}
-		}
-	}
+	buildCache := cache.MakeCache(0, 0)
+	lw := k8sCache.NewListWatchFromClient(pkgw.fissionClient.GetCrdClient(), "packages", apiv1.NamespaceDefault, fields.Everything())
+	_, controller := k8sCache.NewInformer(lw, &crd.Package{}, 60*time.Second, k8sCache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			pkg := obj.(*crd.Package)
+			go pkgw.build(buildCache, pkg)
+		},
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			pkg := newObj.(*crd.Package)
+			go pkgw.build(buildCache, pkg)
+		},
+	})
+	controller.Run(make(chan struct{}))
 }