Add builder manager support (#308)

This change orchestrates function builds.

Environments (in v2) define a builder image, just like they do a runtime image. The builder image contains a build script that's invoked with source and deployment paths (as env vars).

The buildermgr watches for environments with build images defined, and creates build deployments and services.

Functions can define source and deployment. Buildermgr watches for functions with source code (and build status == pending) and invokes the environment's builder when appropriate. It captures logs from the build and sets the build status (success/failure) and build lots into the PackageStatus.

buildermgr/api.go

@@ -0,0 +1,108 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/gorilla/handlers"
+	"github.com/gorilla/mux"
+	"k8s.io/client-go/1.5/kubernetes"
+	"k8s.io/client-go/1.5/pkg/api"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/tpr"
+)
+
+type (
+	BuildRequest struct {
+		Package api.ObjectMeta `json:"package"`
+	}
+
+	BuilderMgr struct {
+		fissionClient    *tpr.FissionClient
+		kubernetesClient *kubernetes.Clientset
+		storageSvcUrl    string
+		namespace        string
+	}
+)
+
+func MakeBuilderMgr(fissionClient *tpr.FissionClient,
+	kubernetesClient *kubernetes.Clientset, storageSvcUrl string,
+	envBuilderNamespace string) *BuilderMgr {
+
+	envWatcher := makeEnvironmentWatcher(fissionClient, kubernetesClient, envBuilderNamespace)
+	go envWatcher.watchEnvironments()
+
+	pkgWatcher := makePackageWatcher(fissionClient, kubernetesClient, envBuilderNamespace, storageSvcUrl)
+	go pkgWatcher.watchPackages()
+
+	return &BuilderMgr{
+		fissionClient:    fissionClient,
+		kubernetesClient: kubernetesClient,
+		storageSvcUrl:    storageSvcUrl,
+		namespace:        envBuilderNamespace,
+	}
+}
+
+func (builderMgr *BuilderMgr) build(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		e := fmt.Sprintf("Failed to read request: %v", err)
+		log.Println(e)
+		http.Error(w, e, 500)
+		return
+	}
+
+	buildReq := BuildRequest{}
+	err = json.Unmarshal([]byte(body), &buildReq)
+	if err != nil {
+		e := fmt.Sprintf("invalid request body: %v", err)
+		log.Println(e)
+		http.Error(w, e, 400)
+		return
+	}
+
+	buildLogs, err := buildPackage(builderMgr.fissionClient, builderMgr.kubernetesClient,
+		builderMgr.namespace, builderMgr.storageSvcUrl, buildReq)
+	if err != nil {
+		code, e := fission.GetHTTPError(err)
+		http.Error(w, e, code)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain")
+	_, err = w.Write([]byte(buildLogs))
+	if err != nil {
+		e := fmt.Sprintf("Failed to reply http request: %v", err)
+		log.Println(e)
+		http.Error(w, e, 500)
+	}
+}
+
+func (builderMgr *BuilderMgr) Serve(port int) {
+	r := mux.NewRouter()
+	r.HandleFunc("/v1/build", builderMgr.build).Methods("POST")
+	address := fmt.Sprintf(":%v", port)
+	log.Printf("Start buildermgr at port %v", address)
+	log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r)))
+}

buildermgr/buildermgr.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"log"
+
+	"github.com/fission/fission/tpr"
+)
+
+// Start the buildermgr service.
+func Start(port int, storageSvcUrl string, envBuilderNamespace string) error {
+	fissionClient, kubernetesClient, err := tpr.MakeFissionClient()
+	if err != nil {
+		log.Printf("Failed to get kubernetes client: %v", err)
+		return err
+	}
+
+	api := MakeBuilderMgr(fissionClient, kubernetesClient,
+		storageSvcUrl, envBuilderNamespace)
+
+	go api.Serve(port)
+
+	return nil
+}

buildermgr/client/client.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/buildermgr"
+)
+
+type (
+	Client struct {
+		url string
+	}
+)
+
+func MakeClient(builderUrl string) *Client {
+	return &Client{
+		url: strings.TrimSuffix(builderUrl, "/") + "/v1",
+	}
+}
+
+func (c *Client) PackageBuild(req *buildermgr.BuildRequest) ([]byte, error) {
+	body, err := json.Marshal(req)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.Post(c.url+"/build", "application/json", bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	return c.handleResponse(resp)
+}
+
+func (c *Client) handleResponse(resp *http.Response) ([]byte, error) {
+	if resp.StatusCode != 200 {
+		return nil, fission.MakeErrorFromHTTP(resp)
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	return body, err
+}

buildermgr/common.go

@@ -0,0 +1,211 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"k8s.io/client-go/1.5/kubernetes"
+	"k8s.io/client-go/1.5/pkg/api"
+
+	"github.com/dchest/uniuri"
+	"github.com/fission/fission"
+	"github.com/fission/fission/builder"
+	builderClient "github.com/fission/fission/builder/client"
+	"github.com/fission/fission/environments/fetcher"
+	fetcherClient "github.com/fission/fission/environments/fetcher/client"
+	"github.com/fission/fission/tpr"
+)
+
+// buildPackage helps to build source package into deployment package.
+// Following is steps buildPackage takes to complete the build process.
+// 1. Check package status
+// 2. Update package status to running state
+// 3. Send fetch request to fetcher to fetch source package.
+// 4. Send build request to builder to start a build.
+// 5. Send upload request to fetcher to upload deployment package.
+// 6. Update package status to succeed state
+// 7. Update package resource in package ref of functions that share the same package
+// *. Update package status to failed state,if any one of steps above failed
+func buildPackage(fissionClient *tpr.FissionClient, kubernetesClient *kubernetes.Clientset,
+	builderNamespace string, storageSvcUrl string, buildReq BuildRequest) (buildLogs string, err error) {
+
+	pkg, err := fissionClient.Packages(
+		buildReq.Package.Namespace).Get(buildReq.Package.Name)
+	if err != nil {
+		e := fmt.Sprintf("Error getting function TPR info: %v", err)
+		log.Println(e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		return e, fission.MakeError(500, e)
+	}
+
+	// Only do build for pending packages
+	if pkg.Status.BuildStatus != fission.BuildStatusPending {
+		e := "package is not in pending state"
+		log.Println(e)
+		return e, fission.MakeError(400, e)
+	}
+
+	// update package status to running state, so that
+	// we can know what status a package is through cli.
+	_, err = updatePackage(fissionClient, pkg, fission.BuildStatusRunning, "", nil)
+	if err != nil {
+		e := fmt.Sprintf("Error setting package pending state: %v", err)
+		log.Println(e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		return e, fission.MakeError(500, e)
+	}
+
+	env, err := fissionClient.Environments(api.NamespaceDefault).Get(pkg.Spec.Environment.Name)
+	if err != nil {
+		e := fmt.Sprintf("Error getting environment TPR info: %v", err)
+		log.Println(e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		return e, fission.MakeError(500, e)
+	}
+
+	svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, builderNamespace)
+	srcPkgFilename := fmt.Sprintf("%v-%v", pkg.Metadata.Name, strings.ToLower(uniuri.NewLen(6)))
+	fetcherC := fetcherClient.MakeClient(fmt.Sprintf("http://%v:8000", svcName))
+	builderC := builderClient.MakeClient(fmt.Sprintf("http://%v:8001", svcName))
+
+	fetchReq := &fetcher.FetchRequest{
+		FetchType: fetcher.FETCH_SOURCE,
+		Package:   pkg.Metadata,
+		Filename:  srcPkgFilename,
+	}
+
+	// send fetch request to fetcher
+	err = fetcherC.Fetch(fetchReq)
+	if err != nil {
+		e := fmt.Sprintf("Error fetching source package: %v", err)
+		log.Println(e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		return e, fission.MakeError(500, e)
+	}
+
+	pkgBuildReq := &builder.PackageBuildRequest{
+		SrcPkgFilename: srcPkgFilename,
+		BuildCommand:   pkg.Spec.BuildCommand,
+	}
+
+	log.Printf("Start building with source package: %v", srcPkgFilename)
+	// send build request to builder
+	buildResp, err := builderC.Build(pkgBuildReq)
+	if err != nil {
+		e := fmt.Sprintf("Error building deployment package: %v", err)
+		log.Println(e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		return e, fission.MakeError(500, e)
+	}
+
+	log.Printf("Build succeed, source package: %v, deployment package: %v", srcPkgFilename, buildResp.ArtifactFilename)
+
+	uploadReq := &fetcher.UploadRequest{
+		Filename:      buildResp.ArtifactFilename,
+		StorageSvcUrl: storageSvcUrl,
+	}
+
+	log.Printf("Start uploading deployment package: %v", buildResp.ArtifactFilename)
+	// ask fetcher to upload the deployment package
+	uploadResp, err := fetcherC.Upload(uploadReq)
+	if err != nil {
+		e := fmt.Sprintf("Error uploading deployment package: %v", err)
+		log.Println(e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		return e, fission.MakeError(500, e)
+	}
+
+	log.Printf("Start updating info of package: %v", pkg.Metadata.Name)
+	// update package status and also build logs
+	newPkgRV, err := updatePackage(fissionClient, pkg,
+		fission.BuildStatusSucceeded, buildResp.BuildLogs, uploadResp)
+	if err != nil {
+		e := fmt.Sprintf("Error creating deployment package TPR resource: %v", err)
+		log.Println(e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		return e, fission.MakeError(500, e)
+	}
+
+	fnList, err := fissionClient.
+		Functions(api.NamespaceDefault).List(api.ListOptions{})
+	if err != nil {
+		e := fmt.Sprintf("Error getting function list: %v", err)
+		log.Println(e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		return e, fission.MakeError(500, e)
+	}
+
+	// A package may be used by multiple functions. Update
+	// functions with old package resource version
+	for _, fn := range fnList.Items {
+		if fn.Spec.Package.PackageRef.Name == pkg.Metadata.Name &&
+			fn.Spec.Package.PackageRef.Namespace == pkg.Metadata.Namespace &&
+			fn.Spec.Package.PackageRef.ResourceVersion != pkg.Metadata.ResourceVersion {
+			fn.Spec.Package.PackageRef.ResourceVersion = newPkgRV
+			// update TPR
+			_, err = fissionClient.Functions(fn.Metadata.Namespace).Update(&fn)
+			if err != nil {
+				e := fmt.Sprintf("Error updating function package resource version: %v", err)
+				log.Println(e)
+				updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+				return e, fission.MakeError(500, e)
+			}
+		}
+	}
+
+	log.Printf("Completed build request for package: %v", pkg.Metadata.Name)
+
+	return buildResp.BuildLogs, nil
+}
+
+func updatePackage(fissionClient *tpr.FissionClient,
+	pkg *tpr.Package, status fission.BuildStatus, buildLogs string,
+	uploadResp *fetcher.UploadResponse) (string, error) {
+
+	// Kubernetes checks resource version before applying
+	// new resource config. The update operation will be
+	// rejected if the resource version in metadata is lower
+	// than the latest version. Set resource version empty
+	// string to skip resource version check.
+	pkg.Metadata.ResourceVersion = ""
+
+	pkg.Status = fission.PackageStatus{
+		BuildStatus: status,
+		BuildLog:    buildLogs,
+	}
+
+	if uploadResp != nil {
+		pkg.Spec.Deployment = fission.Archive{
+			Type:     fission.ArchiveTypeUrl,
+			URL:      uploadResp.ArchiveDownloadUrl,
+			Checksum: uploadResp.Checksum,
+		}
+	}
+
+	// update package spec
+	pkg, err := fissionClient.Packages(api.NamespaceDefault).Update(pkg)
+	if err != nil {
+		log.Printf("Error updating package: %v", err)
+		return "", err
+	}
+
+	// return resource version for function to update function package ref
+	return pkg.Metadata.ResourceVersion, nil
+}

buildermgr/envwatcher.go

@@ -0,0 +1,477 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"k8s.io/client-go/1.5/kubernetes"
+	"k8s.io/client-go/1.5/pkg/api"
+	"k8s.io/client-go/1.5/pkg/api/v1"
+	"k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"
+	"k8s.io/client-go/1.5/pkg/labels"
+	"k8s.io/client-go/1.5/pkg/util/intstr"
+	"k8s.io/client-go/1.5/pkg/watch"
+
+	"github.com/fission/fission/tpr"
+)
+
+type requestType int
+
+const (
+	GET_BUILDER requestType = iota
+	CLEANUP_BUILDERS
+
+	LABEL_ENV_NAME            = "envName"
+	LABEL_ENV_RESOURCEVERSION = "envResourceVersion"
+)
+
+type (
+	builderInfo struct {
+		envMetadata *api.ObjectMeta
+		deployment  *v1beta1.Deployment
+		service     *v1.Service
+	}
+
+	envwRequest struct {
+		requestType
+		env      *tpr.Environment
+		envList  []tpr.Environment
+		respChan chan envwResponse
+	}
+
+	envwResponse struct {
+		builderInfo *builderInfo
+		err         error
+	}
+
+	environmentWatcher struct {
+		cache                  map[string]*builderInfo
+		requestChan            chan envwRequest
+		builderNamespace       string
+		fissionClient          *tpr.FissionClient
+		kubernetesClient       *kubernetes.Clientset
+		fetcherImage           string
+		fetcherImagePullPolicy v1.PullPolicy
+	}
+)
+
+func makeEnvironmentWatcher(fissionClient *tpr.FissionClient,
+	kubernetesClient *kubernetes.Clientset, builderNamespace string) *environmentWatcher {
+
+	fetcherImage := os.Getenv("FETCHER_IMAGE")
+	if len(fetcherImage) == 0 {
+		fetcherImage = "fission/fetcher"
+	}
+
+	fetcherImagePullPolicy := os.Getenv("FETCHER_IMAGE_PULL_POLICY")
+	if len(fetcherImagePullPolicy) == 0 {
+		fetcherImagePullPolicy = "IfNotPresent"
+	}
+
+	var pullPolicy v1.PullPolicy
+	switch fetcherImagePullPolicy {
+	case "Always":
+		pullPolicy = v1.PullAlways
+	case "Never":
+		pullPolicy = v1.PullNever
+	default:
+		pullPolicy = v1.PullIfNotPresent
+	}
+
+	envWatcher := &environmentWatcher{
+		cache:                  make(map[string]*builderInfo),
+		requestChan:            make(chan envwRequest),
+		builderNamespace:       builderNamespace,
+		fissionClient:          fissionClient,
+		kubernetesClient:       kubernetesClient,
+		fetcherImage:           fetcherImage,
+		fetcherImagePullPolicy: pullPolicy,
+	}
+
+	go envWatcher.service()
+
+	return envWatcher
+}
+
+func (envw *environmentWatcher) getCacheKey(envName string, envResourceVersion string) string {
+	return fmt.Sprintf("%v-%v", envName, envResourceVersion)
+}
+
+func (envw *environmentWatcher) getLabels(envName string, envResourceVersion string) map[string]string {
+	return map[string]string{
+		LABEL_ENV_NAME:            envName,
+		LABEL_ENV_RESOURCEVERSION: envResourceVersion,
+	}
+}
+
+func (envw *environmentWatcher) watchEnvironments() {
+	rv := ""
+	for {
+		wi, err := envw.fissionClient.Environments(api.NamespaceAll).Watch(api.ListOptions{
+			ResourceVersion: rv,
+		})
+		if err != nil {
+			log.Fatalf("Error watching environment list: %v", err)
+		}
+
+		for {
+			ev, more := <-wi.ResultChan()
+			if !more {
+				// restart watch from last rv
+				break
+			}
+			if ev.Type == watch.Error {
+				// restart watch from the start
+				rv = ""
+				time.Sleep(time.Second)
+				break
+			}
+			env := ev.Object.(*tpr.Environment)
+			rv = env.Metadata.ResourceVersion
+			envw.sync()
+		}
+	}
+}
+
+func (envw *environmentWatcher) sync() {
+	envList, err := envw.fissionClient.Environments(api.NamespaceAll).List(api.ListOptions{})
+	if err != nil {
+		log.Fatalf("Error syncing environment TPR resources: %v", err)
+	}
+
+	// Create environment builders for all environments
+	for i := range envList.Items {
+		env := envList.Items[i]
+
+		if env.Spec.Version == 1 || // builder is not supported with v1 interface
+			len(env.Spec.Builder.Image) == 0 { // ignore env without builder image
+			continue
+		}
+		_, err := envw.getEnvBuilder(&env)
+		if err != nil {
+			log.Printf("Error creating builder for %v: %v", env.Metadata.Name, err)
+		}
+	}
+	envw.cleanupEnvBuilders(envList.Items)
+}
+
+func (envw *environmentWatcher) service() {
+	for {
+		req := <-envw.requestChan
+		switch req.requestType {
+		case GET_BUILDER:
+			key := envw.getCacheKey(req.env.Metadata.Name, req.env.Metadata.ResourceVersion)
+			builderInfo, ok := envw.cache[key]
+			if !ok {
+				builderInfo, err := envw.createBuilder(req.env)
+				if err != nil {
+					req.respChan <- envwResponse{err: err}
+					continue
+				}
+				envw.cache[key] = builderInfo
+			}
+			req.respChan <- envwResponse{builderInfo: builderInfo}
+
+		case CLEANUP_BUILDERS:
+			latestEnvList := make(map[string]*tpr.Environment)
+			for i := range req.envList {
+				env := req.envList[i]
+				key := envw.getCacheKey(env.Metadata.Name, env.Metadata.ResourceVersion)
+				latestEnvList[key] = &env
+			}
+
+			// If an environment is deleted when builder manager down,
+			// the builder belongs to the environment will be out-of-
+			// control (an orphan builder) since there is no record in
+			// cache and TPR. We need to iterate over the services &
+			// deployments to remove both normal and orphan builders.
+
+			svcList, err := envw.getBuilderServiceList(nil)
+			if err != nil {
+				log.Println(err.Error())
+			}
+			for _, svc := range svcList {
+				envName := svc.ObjectMeta.Labels[LABEL_ENV_NAME]
+				envResourceVersion := svc.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION]
+				key := envw.getCacheKey(envName, envResourceVersion)
+				if _, ok := latestEnvList[key]; !ok {
+					err := envw.deleteBuilderService(svc.ObjectMeta.Labels)
+					if err != nil {
+						log.Printf("Error removing builder service: %v", err)
+					}
+				}
+				delete(envw.cache, svc.ObjectMeta.Name)
+			}
+
+			deployList, err := envw.getBuilderDeploymentList(nil)
+			if err != nil {
+				log.Printf(err.Error())
+			}
+			for _, deploy := range deployList {
+				envName := deploy.ObjectMeta.Labels[LABEL_ENV_NAME]
+				envResourceVersion := deploy.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION]
+				key := envw.getCacheKey(envName, envResourceVersion)
+				if _, ok := latestEnvList[key]; !ok {
+					err := envw.deleteBuilderDeployment(deploy.ObjectMeta.Labels)
+					if err != nil {
+						log.Printf("Error removing builder deployment: %v", err)
+					}
+				}
+				delete(envw.cache, deploy.ObjectMeta.Name)
+			}
+		}
+	}
+}
+
+func (envw *environmentWatcher) getEnvBuilder(env *tpr.Environment) (*builderInfo, error) {
+	respChan := make(chan envwResponse)
+	envw.requestChan <- envwRequest{
+		requestType: GET_BUILDER,
+		env:         env,
+		respChan:    respChan,
+	}
+	resp := <-respChan
+	return resp.builderInfo, resp.err
+}
+
+func (envw *environmentWatcher) cleanupEnvBuilders(envs []tpr.Environment) {
+	envw.requestChan <- envwRequest{
+		requestType: CLEANUP_BUILDERS,
+		envList:     envs,
+	}
+}
+
+func (envw *environmentWatcher) createBuilder(env *tpr.Environment) (*builderInfo, error) {
+	var svc *v1.Service
+	var deploy *v1beta1.Deployment
+
+	sel := envw.getLabels(env.Metadata.Name, env.Metadata.ResourceVersion)
+
+	svcList, err := envw.getBuilderServiceList(sel)
+	if err != nil {
+		return nil, err
+	}
+	if len(svcList) == 0 {
+		svc, err = envw.createBuilderService(env)
+		if err != nil {
+			return nil, fmt.Errorf("Error creating builder service: %v", err)
+		}
+	}
+
+	deployList, err := envw.getBuilderDeploymentList(sel)
+	if err != nil {
+		return nil, err
+	}
+	if len(deployList) == 0 {
+		deploy, err = envw.createBuilderDeployment(env)
+		if err != nil {
+			return nil, fmt.Errorf("Error creating builder deployment: %v", err)
+		}
+	}
+
+	return &builderInfo{
+		envMetadata: &env.Metadata,
+		service:     svc,
+		deployment:  deploy,
+	}, nil
+}
+
+func (envw *environmentWatcher) deleteBuilderService(sel map[string]string) error {
+	svcList, err := envw.getBuilderServiceList(sel)
+	if err != nil {
+		return err
+	}
+	for _, svc := range svcList {
+		log.Printf("Removing builder service: %v", svc.ObjectMeta.Name)
+
+		// cascading deletion
+		// https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/
+		falseVal := false
+		delOpt := &api.DeleteOptions{
+			OrphanDependents: &falseVal,
+		}
+
+		err = envw.kubernetesClient.
+			Services(envw.builderNamespace).
+			Delete(svc.ObjectMeta.Name, delOpt)
+		if err != nil {
+			return fmt.Errorf("Error deleting builder service: %v", err)
+		}
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) deleteBuilderDeployment(sel map[string]string) error {
+	deployList, err := envw.getBuilderDeploymentList(sel)
+	if err != nil {
+		return err
+	}
+	for _, deploy := range deployList {
+		log.Printf("Removing builder deployment: %v", deploy.ObjectMeta.Name)
+
+		falseVal := false
+		delOpt := &api.DeleteOptions{
+			OrphanDependents: &falseVal,
+		}
+
+		err = envw.kubernetesClient.
+			Deployments(envw.builderNamespace).
+			Delete(deploy.ObjectMeta.Name, delOpt)
+		if err != nil {
+			return fmt.Errorf("Error deleteing builder deployment: %v", err)
+		}
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) getBuilderServiceList(sel map[string]string) ([]v1.Service, error) {
+	svcList, err := envw.kubernetesClient.Services(envw.builderNamespace).List(
+		api.ListOptions{
+			LabelSelector: labels.Set(sel).AsSelector(),
+		})
+	if err != nil {
+		return nil, fmt.Errorf("Error getting builder service list: %v", err)
+	}
+	return svcList.Items, nil
+}
+
+func (envw *environmentWatcher) createBuilderService(env *tpr.Environment) (*v1.Service, error) {
+	name := envw.getCacheKey(env.Metadata.Name, env.Metadata.ResourceVersion)
+	sel := envw.getLabels(env.Metadata.Name, env.Metadata.ResourceVersion)
+	service := v1.Service{
+		ObjectMeta: v1.ObjectMeta{
+			Namespace: envw.builderNamespace,
+			Name:      name,
+			Labels:    sel,
+		},
+		Spec: v1.ServiceSpec{
+			Selector: sel,
+			Type:     v1.ServiceTypeClusterIP,
+			Ports: []v1.ServicePort{
+				{
+					Name:     "fetcher-port",
+					Protocol: v1.ProtocolTCP,
+					Port:     8000,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 8000,
+					},
+				},
+				{
+					Name:     "builder-port",
+					Protocol: v1.ProtocolTCP,
+					Port:     8001,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 8001,
+					},
+				},
+			},
+		},
+	}
+	log.Printf("Creating builder service: %v", name)
+	_, err := envw.kubernetesClient.Services(envw.builderNamespace).Create(&service)
+	if err != nil {
+		return nil, err
+	}
+	return &service, nil
+}
+
+func (envw *environmentWatcher) getBuilderDeploymentList(sel map[string]string) ([]v1beta1.Deployment, error) {
+	deployList, err := envw.kubernetesClient.Deployments(envw.builderNamespace).List(
+		api.ListOptions{
+			LabelSelector: labels.Set(sel).AsSelector(),
+		})
+	if err != nil {
+		return nil, fmt.Errorf("Error getting builder deployment list: %v", err)
+	}
+	return deployList.Items, nil
+}
+
+func (envw *environmentWatcher) createBuilderDeployment(env *tpr.Environment) (*v1beta1.Deployment, error) {
+	sharedMountPath := "/package"
+	name := envw.getCacheKey(env.Metadata.Name, env.Metadata.ResourceVersion)
+	sel := envw.getLabels(env.Metadata.Name, env.Metadata.ResourceVersion)
+	var replicas int32 = 1
+	deployment := &v1beta1.Deployment{
+		ObjectMeta: v1.ObjectMeta{
+			Namespace: envw.builderNamespace,
+			Name:      name,
+			Labels:    sel,
+		},
+		Spec: v1beta1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &v1beta1.LabelSelector{
+				MatchLabels: sel,
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: v1.ObjectMeta{
+					Labels: sel,
+				},
+				Spec: v1.PodSpec{
+					Volumes: []v1.Volume{
+						{
+							Name: "package",
+							VolumeSource: v1.VolumeSource{
+								EmptyDir: &v1.EmptyDirVolumeSource{},
+							},
+						},
+					},
+					Containers: []v1.Container{
+						{
+							Name:                   "builder",
+							Image:                  env.Spec.Builder.Image,
+							ImagePullPolicy:        v1.PullAlways,
+							TerminationMessagePath: "/dev/termination-log",
+							VolumeMounts: []v1.VolumeMount{
+								{
+									Name:      "package",
+									MountPath: sharedMountPath,
+								},
+							},
+							Command: []string{"/builder", sharedMountPath},
+						},
+						{
+							Name:                   "fetcher",
+							Image:                  envw.fetcherImage,
+							ImagePullPolicy:        envw.fetcherImagePullPolicy,
+							TerminationMessagePath: "/dev/termination-log",
+							VolumeMounts: []v1.VolumeMount{
+								{
+									Name:      "package",
+									MountPath: sharedMountPath,
+								},
+							},
+							Command: []string{"/fetcher", sharedMountPath},
+						},
+					},
+					ServiceAccountName: "fission-builder",
+				},
+			},
+		},
+	}
+	log.Printf("Creating builder deployment: %v", envw.getCacheKey(env.Metadata.Name, env.Metadata.ResourceVersion))
+	_, err := envw.kubernetesClient.Deployments(envw.builderNamespace).Create(deployment)
+	if err != nil {
+		return nil, err
+	}
+	return deployment, nil
+}

buildermgr/pkgwatcher.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"log"
+	"time"
+
+	"k8s.io/client-go/1.5/kubernetes"
+	"k8s.io/client-go/1.5/pkg/api"
+	"k8s.io/client-go/1.5/pkg/watch"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/tpr"
+)
+
+type (
+	packageWatcher struct {
+		fissionClient    *tpr.FissionClient
+		kubernetesClient *kubernetes.Clientset
+		builderNamespace string
+		storageSvcUrl    string
+	}
+)
+
+func makePackageWatcher(fissionClient *tpr.FissionClient,
+	kubernetesClient *kubernetes.Clientset, builderNamespace string, storageSvcUrl string) *packageWatcher {
+	pkgw := &packageWatcher{
+		fissionClient:    fissionClient,
+		kubernetesClient: kubernetesClient,
+		builderNamespace: builderNamespace,
+		storageSvcUrl:    storageSvcUrl,
+	}
+	return pkgw
+}
+
+func (pkgw *packageWatcher) build(pkgMetadata api.ObjectMeta) {
+	buildReq := BuildRequest{
+		Package: pkgMetadata,
+	}
+	_, err := buildPackage(pkgw.fissionClient,
+		pkgw.kubernetesClient, pkgw.builderNamespace, pkgw.storageSvcUrl, buildReq)
+	if err != nil {
+		log.Printf("Error building package %v: %v", buildReq.Package.Name, err)
+	}
+}
+
+func (pkgw *packageWatcher) watchPackages() {
+	rv := ""
+	for {
+		wi, err := pkgw.fissionClient.Packages(api.NamespaceDefault).Watch(api.ListOptions{
+			ResourceVersion: rv,
+		})
+		if err != nil {
+			log.Fatalf("Error watching package TPR resources: %v", err)
+		}
+
+		for {
+			ev, more := <-wi.ResultChan()
+			if !more {
+				break
+			}
+			if ev.Type == watch.Error {
+				rv = ""
+				time.Sleep(time.Second)
+				break
+			}
+			pkg := ev.Object.(*tpr.Package)
+			rv = pkg.Metadata.ResourceVersion
+
+			// only do build for packages in pending state
+			if pkg.Status.BuildStatus == fission.BuildStatusPending {
+				go pkgw.build(pkg.Metadata)
+			}
+		}
+	}
+}