Move packages to proejct/pkg to follow go project folder structure convention (#1190)

pkg/buildermgr/buildermgr.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/crd"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+)
+
+// Start the buildermgr service.
+func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
+	bmLogger := logger.Named("builder_manager")
+
+	fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to get fission or kubernetes client")
+	}
+
+	err = fissionClient.WaitForCRDs()
+	if err != nil {
+		return errors.Wrap(err, "error waiting for CRDs")
+	}
+
+	fetcherConfig, err := fetcherConfig.MakeFetcherConfig("/packages")
+	if err != nil {
+		return errors.Wrap(err, "error making fetcher config")
+	}
+
+	envWatcher := makeEnvironmentWatcher(bmLogger, fissionClient, kubernetesClient, fetcherConfig, envBuilderNamespace)
+	go envWatcher.watchEnvironments()
+
+	pkgWatcher := makePackageWatcher(bmLogger, fissionClient,
+		kubernetesClient, envBuilderNamespace, storageSvcUrl)
+	go pkgWatcher.watchPackages(fissionClient, kubernetesClient, envBuilderNamespace)
+
+	select {}
+}

pkg/buildermgr/common.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/dchest/uniuri"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/builder"
+	builderClient "github.com/fission/fission/pkg/builder/client"
+	"github.com/fission/fission/pkg/crd"
+	ferror "github.com/fission/fission/pkg/error"
+	fetcherClient "github.com/fission/fission/pkg/fetcher/client"
+	"github.com/fission/fission/pkg/types"
+)
+
+// buildPackage helps to build source package into deployment package.
+// Following is the steps buildPackage function takes to complete the whole process.
+// 1. Send fetch request to fetcher to fetch source package.
+// 2. Send build request to builder to start a build.
+// 3. Send upload request to fetcher to upload deployment package.
+// 4. Return upload response and build logs.
+// *. Return build logs and error if any one of steps above failed.
+func buildPackage(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, envBuilderNamespace string,
+	storageSvcUrl string, pkg *fv1.Package) (uploadResp *types.ArchiveUploadResponse, buildLogs string, err error) {
+
+	env, err := fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
+	if err != nil {
+		e := "error getting environment CRD info"
+		logger.Error(e, zap.Error(err))
+		e = fmt.Sprintf("%s: %v", e, err)
+		return nil, e, ferror.MakeError(http.StatusInternalServerError, e)
+	}
+
+	svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, envBuilderNamespace)
+	srcPkgFilename := fmt.Sprintf("%v-%v", pkg.Metadata.Name, strings.ToLower(uniuri.NewLen(6)))
+	fetcherC := fetcherClient.MakeClient(logger, fmt.Sprintf("http://%v:8000", svcName))
+	builderC := builderClient.MakeClient(logger, fmt.Sprintf("http://%v:8001", svcName))
+
+	fetchReq := &types.FunctionFetchRequest{
+		FetchType:   types.FETCH_SOURCE,
+		Package:     pkg.Metadata,
+		Filename:    srcPkgFilename,
+		KeepArchive: false,
+	}
+
+	// send fetch request to fetcher
+	err = fetcherC.Fetch(ctx, fetchReq)
+	if err != nil {
+		e := "error fetching source package"
+		logger.Error(e, zap.Error(err))
+		e = fmt.Sprintf("%s: %v", e, err)
+		return nil, e, ferror.MakeError(http.StatusInternalServerError, e)
+	}
+
+	buildCmd := pkg.Spec.BuildCommand
+	if len(buildCmd) == 0 {
+		buildCmd = env.Spec.Builder.Command
+	}
+
+	pkgBuildReq := &builder.PackageBuildRequest{
+		SrcPkgFilename: srcPkgFilename,
+		BuildCommand:   buildCmd,
+	}
+
+	logger.Info("started building with source package", zap.String("source_package", srcPkgFilename))
+	// send build request to builder
+	buildResp, err := builderC.Build(pkgBuildReq)
+	if err != nil {
+		e := fmt.Sprintf("Error building deployment package: %v", err)
+		var buildLogs string
+		if buildResp != nil {
+			buildLogs = buildResp.BuildLogs
+		}
+		buildLogs += fmt.Sprintf("%v\n", e)
+		return nil, buildLogs, ferror.MakeError(http.StatusInternalServerError, e)
+	}
+
+	logger.Info("build succeed", zap.String("source_package", srcPkgFilename), zap.String("deployment_package", buildResp.ArtifactFilename))
+
+	archivePackage := !env.Spec.KeepArchive
+
+	uploadReq := &types.ArchiveUploadRequest{
+		Filename:       buildResp.ArtifactFilename,
+		StorageSvcUrl:  storageSvcUrl,
+		ArchivePackage: archivePackage,
+	}
+
+	logger.Info("started uploading deployment package", zap.String("deployment_package", buildResp.ArtifactFilename))
+	// ask fetcher to upload the deployment package
+	uploadResp, err = fetcherC.Upload(ctx, uploadReq)
+	if err != nil {
+		e := fmt.Sprintf("Error uploading deployment package: %v", err)
+		buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
+		return nil, buildResp.BuildLogs, ferror.MakeError(http.StatusInternalServerError, e)
+	}
+
+	return uploadResp, buildResp.BuildLogs, nil
+}
+
+func updatePackage(logger *zap.Logger, fissionClient *crd.FissionClient,
+	pkg *fv1.Package, status fv1.BuildStatus, buildLogs string,
+	uploadResp *types.ArchiveUploadResponse) (*fv1.Package, error) {
+
+	pkg.Status = fv1.PackageStatus{
+		BuildStatus: status,
+		BuildLog:    buildLogs,
+	}
+
+	if uploadResp != nil {
+		pkg.Spec.Deployment = fv1.Archive{
+			Type:     types.ArchiveTypeUrl,
+			URL:      uploadResp.ArchiveDownloadUrl,
+			Checksum: uploadResp.Checksum,
+		}
+	}
+
+	// update package spec
+	pkg, err := fissionClient.Packages(pkg.Metadata.Namespace).Update(pkg)
+	if err != nil {
+		e := "error updating package"
+		logger.Error(e, zap.Error(err))
+		return nil, errors.Wrap(err, e)
+	}
+
+	// return resource version for function to update function package ref
+	return pkg, nil
+}

pkg/buildermgr/envwatcher.go

@@ -0,0 +1,560 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/executor/util"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+)
+
+type requestType int
+
+const (
+	GET_BUILDER requestType = iota
+	CLEANUP_BUILDERS
+
+	LABEL_ENV_NAME            = "envName"
+	LABEL_ENV_NAMESPACE       = "envNamespace"
+	LABEL_ENV_RESOURCEVERSION = "envResourceVersion"
+	LABEL_DEPLOYMENT_OWNER    = "owner"
+	BUILDER_MGR               = "buildermgr"
+)
+
+var (
+	deletePropagation = metav1.DeletePropagationBackground
+	delOpt            = metav1.DeleteOptions{PropagationPolicy: &deletePropagation}
+)
+
+type (
+	builderInfo struct {
+		envMetadata *metav1.ObjectMeta
+		deployment  *v1beta1.Deployment
+		service     *apiv1.Service
+	}
+
+	envwRequest struct {
+		requestType
+		env      *fv1.Environment
+		envList  []fv1.Environment
+		respChan chan envwResponse
+	}
+
+	envwResponse struct {
+		builderInfo *builderInfo
+		err         error
+	}
+
+	environmentWatcher struct {
+		logger                 *zap.Logger
+		cache                  map[string]*builderInfo
+		requestChan            chan envwRequest
+		builderNamespace       string
+		fissionClient          *crd.FissionClient
+		kubernetesClient       *kubernetes.Clientset
+		fetcherConfig          *fetcherConfig.Config
+		builderImagePullPolicy apiv1.PullPolicy
+		useIstio               bool
+		collectorEndpoint      string
+	}
+)
+
+func makeEnvironmentWatcher(
+	logger *zap.Logger,
+	fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset,
+	fetcherConfig *fetcherConfig.Config,
+	builderNamespace string) *environmentWatcher {
+
+	useIstio := false
+	enableIstio := os.Getenv("ENABLE_ISTIO")
+	if len(enableIstio) > 0 {
+		istio, err := strconv.ParseBool(enableIstio)
+		if err != nil {
+			logger.Info("Failed to parse ENABLE_ISTIO, defaults to false")
+		}
+		useIstio = istio
+	}
+
+	builderImagePullPolicy := utils.GetImagePullPolicy(os.Getenv("BUILDER_IMAGE_PULL_POLICY"))
+
+	envWatcher := &environmentWatcher{
+		logger:                 logger.Named("environment_watcher"),
+		cache:                  make(map[string]*builderInfo),
+		requestChan:            make(chan envwRequest),
+		builderNamespace:       builderNamespace,
+		fissionClient:          fissionClient,
+		kubernetesClient:       kubernetesClient,
+		builderImagePullPolicy: builderImagePullPolicy,
+		useIstio:               useIstio,
+		fetcherConfig:          fetcherConfig,
+	}
+
+	go envWatcher.service()
+
+	return envWatcher
+}
+
+func (envw *environmentWatcher) getCacheKey(envName string, envNamespace string, envResourceVersion string) string {
+	return fmt.Sprintf("%v-%v-%v", envName, envNamespace, envResourceVersion)
+}
+
+func (env *environmentWatcher) getLabelForDeploymentOwner() map[string]string {
+	return map[string]string{
+		LABEL_DEPLOYMENT_OWNER: BUILDER_MGR,
+	}
+}
+
+func (envw *environmentWatcher) getLabels(envName string, envNamespace string, envResourceVersion string) map[string]string {
+	return map[string]string{
+		LABEL_ENV_NAME:            envName,
+		LABEL_ENV_NAMESPACE:       envNamespace,
+		LABEL_ENV_RESOURCEVERSION: envResourceVersion,
+		LABEL_DEPLOYMENT_OWNER:    BUILDER_MGR,
+	}
+}
+
+func (envw *environmentWatcher) watchEnvironments() {
+	rv := ""
+	for {
+		wi, err := envw.fissionClient.Environments(metav1.NamespaceAll).Watch(metav1.ListOptions{
+			ResourceVersion: rv,
+		})
+		if err != nil {
+			if utils.IsNetworkError(err) {
+				envw.logger.Error("encountered network error, retrying later", zap.Error(err))
+				time.Sleep(5 * time.Second)
+				continue
+			}
+			envw.logger.Fatal("error watching environment list", zap.Error(err))
+		}
+
+		for {
+			ev, more := <-wi.ResultChan()
+			if !more {
+				// restart watch from last rv
+				break
+			}
+			if ev.Type == watch.Error {
+				// restart watch from the start
+				rv = ""
+				time.Sleep(time.Second)
+				break
+			}
+			env := ev.Object.(*fv1.Environment)
+			rv = env.Metadata.ResourceVersion
+			envw.sync()
+		}
+	}
+}
+
+func (envw *environmentWatcher) sync() {
+	maxRetries := 10
+	for i := 0; i < maxRetries; i++ {
+		envList, err := envw.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
+		if err != nil {
+			if utils.IsNetworkError(err) {
+				envw.logger.Error("error syncing environment CRD resources due to network error, retrying later", zap.Error(err))
+				time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
+				continue
+			}
+			envw.logger.Fatal("error syncing environment CRD resources", zap.Error(err))
+		}
+
+		// Create environment builders for all environments
+		for i := range envList.Items {
+			env := envList.Items[i]
+
+			if env.Spec.Version == 1 || // builder is not supported with v1 interface
+				len(env.Spec.Builder.Image) == 0 { // ignore env without builder image
+				continue
+			}
+			_, err := envw.getEnvBuilder(&env)
+			if err != nil {
+				envw.logger.Error("error creating builder", zap.Error(err), zap.String("builder_target", env.Metadata.Name))
+			}
+		}
+
+		// Remove environment builders no longer needed
+		envw.cleanupEnvBuilders(envList.Items)
+		break
+	}
+}
+
+func (envw *environmentWatcher) service() {
+	for {
+		req := <-envw.requestChan
+		switch req.requestType {
+		case GET_BUILDER:
+			// In order to support backward compatibility, for all environments with builder image created in default env,
+			// the pods will be created in fission-builder namespace
+			ns := envw.builderNamespace
+			if req.env.Metadata.Namespace != metav1.NamespaceDefault {
+				ns = req.env.Metadata.Namespace
+			}
+
+			key := envw.getCacheKey(req.env.Metadata.Name, ns, req.env.Metadata.ResourceVersion)
+			builderInfo, ok := envw.cache[key]
+			if !ok {
+				builderInfo, err := envw.createBuilder(req.env, ns)
+				if err != nil {
+					req.respChan <- envwResponse{err: err}
+					continue
+				}
+				envw.cache[key] = builderInfo
+			}
+			req.respChan <- envwResponse{builderInfo: builderInfo}
+
+		case CLEANUP_BUILDERS:
+			latestEnvList := make(map[string]*fv1.Environment)
+			for i := range req.envList {
+				env := req.envList[i]
+				// In order to support backward compatibility, for all builder images created in default
+				// env, the pods are created in fission-builder namespace
+				ns := envw.builderNamespace
+				if env.Metadata.Namespace != metav1.NamespaceDefault {
+					ns = env.Metadata.Namespace
+				}
+				key := envw.getCacheKey(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
+				latestEnvList[key] = &env
+			}
+
+			// If an environment is deleted when builder manager down,
+			// the builder belongs to the environment will be out-of-
+			// control (an orphan builder) since there is no record in
+			// cache and CRD. We need to iterate over the services &
+			// deployments to remove both normal and orphan builders.
+
+			svcList, err := envw.getBuilderServiceList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
+			if err != nil {
+				envw.logger.Error("error getting the builder service list", zap.Error(err))
+			}
+			for _, svc := range svcList {
+				envName := svc.ObjectMeta.Labels[LABEL_ENV_NAME]
+				envNamespace := svc.ObjectMeta.Labels[LABEL_ENV_NAMESPACE]
+				envResourceVersion := svc.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION]
+				key := envw.getCacheKey(envName, envNamespace, envResourceVersion)
+				if _, ok := latestEnvList[key]; !ok {
+					err := envw.deleteBuilderServiceByName(svc.ObjectMeta.Name, svc.ObjectMeta.Namespace)
+					if err != nil {
+						envw.logger.Error("error removing builder service", zap.Error(err),
+							zap.String("service_name", svc.ObjectMeta.Name),
+							zap.String("service_namespace", svc.ObjectMeta.Namespace))
+					}
+				}
+				delete(envw.cache, key)
+			}
+
+			deployList, err := envw.getBuilderDeploymentList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
+			if err != nil {
+				envw.logger.Error("error getting the builder deployment list", zap.Error(err))
+			}
+			for _, deploy := range deployList {
+				envName := deploy.ObjectMeta.Labels[LABEL_ENV_NAME]
+				envNamespace := deploy.ObjectMeta.Labels[LABEL_ENV_NAMESPACE]
+				envResourceVersion := deploy.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION]
+				key := envw.getCacheKey(envName, envNamespace, envResourceVersion)
+				if _, ok := latestEnvList[key]; !ok {
+					err := envw.deleteBuilderDeploymentByName(deploy.ObjectMeta.Name, deploy.ObjectMeta.Namespace)
+					if err != nil {
+						envw.logger.Error("error removing builder deployment", zap.Error(err),
+							zap.String("deployment_name", deploy.ObjectMeta.Name),
+							zap.String("deployment_namespace", deploy.ObjectMeta.Namespace))
+					}
+				}
+				delete(envw.cache, key)
+			}
+		}
+	}
+}
+
+func (envw *environmentWatcher) getEnvBuilder(env *fv1.Environment) (*builderInfo, error) {
+	respChan := make(chan envwResponse)
+	envw.requestChan <- envwRequest{
+		requestType: GET_BUILDER,
+		env:         env,
+		respChan:    respChan,
+	}
+	resp := <-respChan
+	return resp.builderInfo, resp.err
+}
+
+func (envw *environmentWatcher) cleanupEnvBuilders(envs []fv1.Environment) {
+	envw.requestChan <- envwRequest{
+		requestType: CLEANUP_BUILDERS,
+		envList:     envs,
+	}
+}
+
+func (envw *environmentWatcher) createBuilder(env *fv1.Environment, ns string) (*builderInfo, error) {
+	var svc *apiv1.Service
+	var deploy *v1beta1.Deployment
+
+	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
+
+	svcList, err := envw.getBuilderServiceList(sel, ns)
+	if err != nil {
+		return nil, err
+	}
+	// there should be only one service in svcList
+	if len(svcList) == 0 {
+		svc, err = envw.createBuilderService(env, ns)
+		if err != nil {
+			return nil, errors.Wrap(err, "error creating builder service")
+		}
+	} else if len(svcList) == 1 {
+		svc = &svcList[0]
+	} else {
+		return nil, fmt.Errorf("found more than one builder service for environment %q", env.Metadata.Name)
+	}
+
+	deployList, err := envw.getBuilderDeploymentList(sel, ns)
+	if err != nil {
+		return nil, err
+	}
+	// there should be only one deploy in deployList
+	if len(deployList) == 0 {
+		// create builder SA in this ns, if not already created
+		_, err := utils.SetupSA(envw.kubernetesClient, types.FissionBuilderSA, ns)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error creating %q in ns: %s", types.FissionBuilderSA, ns)
+		}
+
+		deploy, err = envw.createBuilderDeployment(env, ns)
+		if err != nil {
+			return nil, errors.Wrap(err, "error creating builder deployment")
+		}
+	} else if len(deployList) == 1 {
+		deploy = &deployList[0]
+	} else {
+		return nil, fmt.Errorf("found more than one builder deployment for environment %q", env.Metadata.Name)
+	}
+
+	return &builderInfo{
+		envMetadata: &env.Metadata,
+		service:     svc,
+		deployment:  deploy,
+	}, nil
+}
+
+func (envw *environmentWatcher) deleteBuilderServiceByName(name, namespace string) error {
+	err := envw.kubernetesClient.CoreV1().
+		Services(namespace).
+		Delete(name, &delOpt)
+	if err != nil {
+		return errors.Wrapf(err, "error deleting builder service %s.%s", name, namespace)
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) deleteBuilderDeploymentByName(name, namespace string) error {
+	err := envw.kubernetesClient.ExtensionsV1beta1().
+		Deployments(namespace).
+		Delete(name, &delOpt)
+	if err != nil {
+		return errors.Wrapf(err, "error deleting builder deployment %s.%s", name, namespace)
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) deleteBuilderService(sel map[string]string, ns string) error {
+	svcList, err := envw.getBuilderServiceList(sel, ns)
+	if err != nil {
+		return err
+	}
+	for _, svc := range svcList {
+		envw.logger.Info("removing builder service", zap.String("service_name", svc.ObjectMeta.Name))
+		err = envw.kubernetesClient.CoreV1().
+			Services(ns).
+			Delete(svc.ObjectMeta.Name, &delOpt)
+		if err != nil {
+			return errors.Wrap(err, "error deleting builder service")
+		}
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) deleteBuilderDeployment(sel map[string]string, ns string) error {
+	deployList, err := envw.getBuilderDeploymentList(sel, ns)
+	if err != nil {
+		return err
+	}
+	for _, deploy := range deployList {
+		envw.logger.Info("removing builder deployment", zap.String("deployment_name", deploy.ObjectMeta.Name))
+		err = envw.kubernetesClient.ExtensionsV1beta1().
+			Deployments(ns).
+			Delete(deploy.ObjectMeta.Name, &delOpt)
+		if err != nil {
+			return errors.Wrap(err, "error deleteing builder deployment")
+		}
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) getBuilderServiceList(sel map[string]string, ns string) ([]apiv1.Service, error) {
+	svcList, err := envw.kubernetesClient.CoreV1().Services(ns).List(
+		metav1.ListOptions{
+			LabelSelector: labels.Set(sel).AsSelector().String(),
+		})
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting builder service list")
+	}
+	return svcList.Items, nil
+}
+
+func (envw *environmentWatcher) createBuilderService(env *fv1.Environment, ns string) (*apiv1.Service, error) {
+	name := fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion)
+	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
+	service := apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns,
+			Name:      name,
+			Labels:    sel,
+		},
+		Spec: apiv1.ServiceSpec{
+			Selector: sel,
+			Type:     apiv1.ServiceTypeClusterIP,
+			Ports: []apiv1.ServicePort{
+				{
+					Name:     "fetcher-port",
+					Protocol: apiv1.ProtocolTCP,
+					Port:     8000,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 8000,
+					},
+				},
+				{
+					Name:     "builder-port",
+					Protocol: apiv1.ProtocolTCP,
+					Port:     8001,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 8001,
+					},
+				},
+			},
+		},
+	}
+	envw.logger.Info("creating builder service", zap.String("service_name", name))
+	_, err := envw.kubernetesClient.CoreV1().Services(ns).Create(&service)
+	if err != nil {
+		return nil, err
+	}
+	return &service, nil
+}
+
+func (envw *environmentWatcher) getBuilderDeploymentList(sel map[string]string, ns string) ([]v1beta1.Deployment, error) {
+	deployList, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).List(
+		metav1.ListOptions{
+			LabelSelector: labels.Set(sel).AsSelector().String(),
+		})
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting builder deployment list")
+	}
+	return deployList.Items, nil
+}
+
+func (envw *environmentWatcher) createBuilderDeployment(env *fv1.Environment, ns string) (*v1beta1.Deployment, error) {
+	name := fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion)
+	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
+	var replicas int32 = 1
+
+	podAnnotations := env.Metadata.Annotations
+	if podAnnotations == nil {
+		podAnnotations = make(map[string]string)
+	}
+	if envw.useIstio && env.Spec.AllowAccessToExternalNetwork {
+		podAnnotations["sidecar.istio.io/inject"] = "false"
+	}
+
+	deployment := &v1beta1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns,
+			Name:      name,
+			Labels:    sel,
+		},
+		Spec: v1beta1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: sel,
+			},
+			Template: apiv1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      sel,
+					Annotations: podAnnotations,
+				},
+				Spec: apiv1.PodSpec{
+					Containers: []apiv1.Container{
+						util.MergeContainerSpecs(&apiv1.Container{
+							Name:                   "builder",
+							Image:                  env.Spec.Builder.Image,
+							ImagePullPolicy:        envw.builderImagePullPolicy,
+							TerminationMessagePath: "/dev/termination-log",
+							Command:                []string{"/builder", envw.fetcherConfig.SharedMountPath()},
+							ReadinessProbe: &apiv1.Probe{
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       2,
+								Handler: apiv1.Handler{
+									HTTPGet: &apiv1.HTTPGetAction{
+										Path: "/healthz",
+										Port: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: 8001,
+										},
+									},
+								},
+							},
+						}, env.Spec.Builder.Container),
+					},
+					ServiceAccountName: "fission-builder",
+				},
+			},
+		},
+	}
+
+	err := envw.fetcherConfig.AddFetcherToPodSpec(&deployment.Spec.Template.Spec, "builder")
+	if err != nil {
+		return nil, err
+	}
+
+	envw.logger.Info("creating builder deployment", zap.String("deployment", name))
+	_, err = envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Create(deployment)
+	if err != nil {
+		return nil, err
+	}
+
+	return deployment, nil
+}

pkg/buildermgr/pkgwatcher.go

@@ -0,0 +1,250 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/cache"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+)
+
+type (
+	packageWatcher struct {
+		logger           *zap.Logger
+		fissionClient    *crd.FissionClient
+		k8sClient        *kubernetes.Clientset
+		podStore         k8sCache.Store
+		pkgStore         k8sCache.Store
+		builderNamespace string
+		storageSvcUrl    string
+	}
+)
+
+func makePackageWatcher(logger *zap.Logger, fissionClient *crd.FissionClient, k8sClientSet *kubernetes.Clientset,
+	builderNamespace string, storageSvcUrl string) *packageWatcher {
+	lw := k8sCache.NewListWatchFromClient(k8sClientSet.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.Everything())
+	store, controller := k8sCache.NewInformer(lw, &apiv1.Pod{}, 30*time.Second, k8sCache.ResourceEventHandlerFuncs{})
+	go controller.Run(make(chan struct{}))
+
+	pkgw := &packageWatcher{
+		logger:           logger.Named("package_watcher"),
+		fissionClient:    fissionClient,
+		k8sClient:        k8sClientSet,
+		podStore:         store,
+		builderNamespace: builderNamespace,
+		storageSvcUrl:    storageSvcUrl,
+	}
+	return pkgw
+}
+
+// build helps to update package status, checks environment builder pod status and
+// dispatches buildPackage to build source package into deployment package.
+// Following is the steps build function takes to complete the whole process.
+// 1. Check package status
+// 2. Update package status to running state
+// 3. Check environment builder pod status
+// 4. Call buildPackage to build package
+// 5. Update package resource in package ref of functions that share the same package
+// 6. Update package status to succeed state
+// *. Update package status to failed state,if any one of steps above failed/time out
+func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *fv1.Package) {
+
+	// Ignore non-pending state packages.
+	if srcpkg.Status.BuildStatus != fv1.BuildStatusPending {
+		return
+	}
+
+	// Ignore duplicate build requests
+	key := fmt.Sprintf("%v-%v", srcpkg.Metadata.Name, srcpkg.Metadata.ResourceVersion)
+	err, _ := buildCache.Set(key, srcpkg)
+	if err != nil {
+		return
+	}
+	defer buildCache.Delete(key)
+
+	pkgw.logger.Info("starting build for package", zap.String("package_name", srcpkg.Metadata.Name), zap.String("resource_version", srcpkg.Metadata.ResourceVersion))
+
+	pkg, err := updatePackage(pkgw.logger, pkgw.fissionClient, srcpkg, fv1.BuildStatusRunning, "", nil)
+	if err != nil {
+		pkgw.logger.Error("error setting package pending state", zap.Error(err))
+		return
+	}
+
+	env, err := pkgw.fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
+	if k8serrors.IsNotFound(err) {
+		e := "environment does not exist"
+		pkgw.logger.Error(e, zap.String("environment", pkg.Spec.Environment.Name))
+		updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
+			fv1.BuildStatusFailed, fmt.Sprintf("%s: %q", e, pkg.Spec.Environment.Name), nil)
+		return
+	}
+
+	// Do health check for environment builder pod
+	for i := 0; i < 15; i++ {
+		// Informer store is not able to use label to find the pod,
+		// iterate all available environment builders.
+		items := pkgw.podStore.List()
+		if err != nil {
+			pkgw.logger.Error("error retrieving pod information for environment", zap.Error(err), zap.String("environment", env.Metadata.Name))
+			return
+		}
+
+		if len(items) == 0 {
+			pkgw.logger.Info("builder pod does not exist for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name))
+			time.Sleep(time.Duration(i*1) * time.Second)
+			continue
+		}
+
+		for _, item := range items {
+			pod := item.(*apiv1.Pod)
+
+			// In order to support backward compatibility, for all builder images created in default env,
+			// the pods will be created in fission-builder namespace
+			builderNs := pkgw.builderNamespace
+			if env.Metadata.Namespace != metav1.NamespaceDefault {
+				builderNs = env.Metadata.Namespace
+			}
+
+			// Filter non-matching pods
+			if pod.ObjectMeta.Labels[LABEL_ENV_NAME] != env.Metadata.Name ||
+				pod.ObjectMeta.Labels[LABEL_ENV_NAMESPACE] != builderNs ||
+				pod.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION] != env.Metadata.ResourceVersion {
+				continue
+			}
+
+			// Pod may become "Running" state but still failed at health check, so use
+			// pod.Status.ContainerStatuses instead of pod.Status.Phase to check pod readiness states.
+			podIsReady := true
+
+			for _, cStatus := range pod.Status.ContainerStatuses {
+				podIsReady = podIsReady && cStatus.Ready
+			}
+
+			if !podIsReady {
+				pkgw.logger.Info("builder pod is not ready for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name))
+				time.Sleep(time.Duration(i*1) * time.Second)
+				break
+			}
+
+			// Add the package getter rolebinding to builder sa
+			// we continue here if role binding was not setup succeesffully. this is because without this, the fetcher wont be able to fetch the source pkg into the container and
+			// the build will fail eventually
+			err := utils.SetupRoleBinding(pkgw.logger, pkgw.k8sClient, types.PackageGetterRB, pkg.Metadata.Namespace, types.PackageGetterCR, types.ClusterRole, types.FissionBuilderSA, builderNs)
+			if err != nil {
+				pkgw.logger.Error("error setting up role binding for package",
+					zap.Error(err),
+					zap.String("role_binding", types.PackageGetterRB),
+					zap.String("package_name", pkg.Metadata.Name),
+					zap.String("package_namespace", pkg.Metadata.Namespace))
+				continue
+			} else {
+				pkgw.logger.Info("setup rolebinding for sa package",
+					zap.String("sa", fmt.Sprintf("%s.%s", types.FissionBuilderSA, builderNs)),
+					zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)))
+			}
+
+			ctx := context.Background()
+			uploadResp, buildLogs, err := buildPackage(ctx, pkgw.logger, pkgw.fissionClient, builderNs, pkgw.storageSvcUrl, pkg)
+			if err != nil {
+				pkgw.logger.Error("error building package", zap.Error(err), zap.String("package_name", pkg.Metadata.Name))
+				updatePackage(pkgw.logger, pkgw.fissionClient, pkg, types.BuildStatusFailed, buildLogs, nil)
+				return
+			}
+
+			pkgw.logger.Info("starting package info update", zap.String("package_name", pkg.Metadata.Name))
+
+			fnList, err := pkgw.fissionClient.
+				Functions(metav1.NamespaceAll).List(metav1.ListOptions{})
+			if err != nil {
+				e := "error getting function list"
+				pkgw.logger.Error(e, zap.Error(err))
+				buildLogs += fmt.Sprintf("%s: %v\n", e, err)
+				updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fv1.BuildStatusFailed, buildLogs, nil)
+			}
+
+			// A package may be used by multiple functions. Update
+			// functions with old package resource version
+			for _, fn := range fnList.Items {
+				if fn.Spec.Package.PackageRef.Name == pkg.Metadata.Name &&
+					fn.Spec.Package.PackageRef.Namespace == pkg.Metadata.Namespace &&
+					fn.Spec.Package.PackageRef.ResourceVersion != pkg.Metadata.ResourceVersion {
+					fn.Spec.Package.PackageRef.ResourceVersion = pkg.Metadata.ResourceVersion
+					// update CRD
+					_, err = pkgw.fissionClient.Functions(fn.Metadata.Namespace).Update(&fn)
+					if err != nil {
+						e := "error updating function package resource version"
+						pkgw.logger.Error(e, zap.Error(err))
+						buildLogs += fmt.Sprintf("%s: %v\n", e, err)
+						updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fv1.BuildStatusFailed, buildLogs, nil)
+						return
+					}
+				}
+			}
+
+			_, err = updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
+				types.BuildStatusSucceeded, buildLogs, uploadResp)
+			if err != nil {
+				pkgw.logger.Error("error updating package info", zap.Error(err), zap.String("package_name", pkg.Metadata.Name))
+				updatePackage(pkgw.logger, pkgw.fissionClient, pkg, types.BuildStatusFailed, buildLogs, nil)
+				return
+			}
+
+			pkgw.logger.Info("completed package build request", zap.String("package_name", pkg.Metadata.Name))
+			return
+		}
+	}
+	// build timeout
+	updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
+		types.BuildStatusFailed, "Build timeout due to environment builder not ready", nil)
+
+	pkgw.logger.Error("max retries exceeded in building source package, timeout due to environment builder not ready",
+		zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)))
+
+	return
+}
+
+func (pkgw *packageWatcher) watchPackages(fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset, builderNamespace string) {
+	buildCache := cache.MakeCache(0, 0)
+	lw := k8sCache.NewListWatchFromClient(pkgw.fissionClient.GetCrdClient(), "packages", apiv1.NamespaceAll, fields.Everything())
+	pkgStore, controller := k8sCache.NewInformer(lw, &fv1.Package{}, 60*time.Second, k8sCache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			pkg := obj.(*fv1.Package)
+			go pkgw.build(buildCache, pkg)
+		},
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			pkg := newObj.(*fv1.Package)
+			go pkgw.build(buildCache, pkg)
+		},
+	})
+	pkgw.pkgStore = pkgStore
+	controller.Run(make(chan struct{}))
+}