Move packages to proejct/pkg to follow go project folder structure convention (#1190)

pkg/builder/builder.go

@@ -0,0 +1,221 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package builder
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/dchest/uniuri"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/info"
+)
+
+const (
+	// supported environment variables
+	envSrcPkg    = "SRC_PKG"
+	envDeployPkg = "DEPLOY_PKG"
+)
+
+type (
+	PackageBuildRequest struct {
+		SrcPkgFilename string `json:"srcPkgFilename"`
+		// Command for builder to run with.
+		// A build command consists of commands, parameters and environment variables.
+		// For now, two environment variables are supported:
+		// 1. SRC_PKG: path to source package directory
+		// 2. DEPLOY_PKG: path to deployment package directory
+		BuildCommand string `json:"command"`
+	}
+
+	PackageBuildResponse struct {
+		ArtifactFilename string `json:"artifactFilename"`
+		BuildLogs        string `json:"buildLogs"`
+	}
+
+	Builder struct {
+		logger           *zap.Logger
+		sharedVolumePath string
+	}
+)
+
+func MakeBuilder(logger *zap.Logger, sharedVolumePath string) *Builder {
+	return &Builder{
+		logger:           logger.Named("builder"),
+		sharedVolumePath: sharedVolumePath,
+	}
+}
+
+func (builder *Builder) VersionHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	fmt.Fprintf(w, info.BuildInfo().String())
+}
+
+func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		e := "method not allowed"
+		builder.logger.Error(e, zap.String("http_method", r.Method))
+		builder.reply(w, "", fmt.Sprintf("%s: %s", e, r.Method), http.StatusMethodNotAllowed)
+		return
+	}
+
+	startTime := time.Now()
+	defer func() {
+		elapsed := time.Since(startTime)
+		builder.logger.Info("build request complete", zap.Duration("elapsed_time", elapsed))
+	}()
+
+	// parse request
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		e := "error reading request body"
+		builder.logger.Error(e, zap.Error(err))
+		builder.reply(w, "", fmt.Sprintf("%s: %s", e, err.Error()), http.StatusInternalServerError)
+		return
+	}
+	var req PackageBuildRequest
+	err = json.Unmarshal(body, &req)
+	if err != nil {
+		e := "error parsing json body"
+		builder.logger.Error(e, zap.Error(err))
+		builder.reply(w, "", fmt.Sprintf("%s: %s", e, err.Error()), http.StatusBadRequest)
+		return
+	}
+	builder.logger.Info("builder received request", zap.Any("request", req))
+
+	builder.logger.Info("starting build")
+	srcPkgPath := filepath.Join(builder.sharedVolumePath, req.SrcPkgFilename)
+	deployPkgFilename := fmt.Sprintf("%v-%v", req.SrcPkgFilename, strings.ToLower(uniuri.NewLen(6)))
+	deployPkgPath := filepath.Join(builder.sharedVolumePath, deployPkgFilename)
+	buildCmd := req.BuildCommand
+	if len(buildCmd) == 0 {
+		// use default build command
+		buildCmd = "/build"
+	}
+	buildLogs, err := builder.build(buildCmd, srcPkgPath, deployPkgPath)
+	if err != nil {
+		e := "error building source package"
+		builder.logger.Error(e, zap.Error(err))
+
+		// append error at the end of build logs
+		buildLogs += fmt.Sprintf("%s: %s\n", e, err.Error())
+		builder.reply(w, deployPkgFilename, buildLogs, http.StatusInternalServerError)
+		return
+	}
+
+	builder.reply(w, deployPkgFilename, buildLogs, http.StatusOK)
+}
+
+func (builder *Builder) reply(w http.ResponseWriter, pkgFilename string, buildLogs string, statusCode int) {
+	resp := PackageBuildResponse{
+		ArtifactFilename: pkgFilename,
+		BuildLogs:        buildLogs,
+	}
+
+	rBody, err := json.Marshal(resp)
+	if err != nil {
+		e := errors.Wrap(err, "error encoding response body")
+		rBody = []byte(fmt.Sprintf(`{"buildLogs": "%v"}`, e.Error()))
+		statusCode = http.StatusInternalServerError
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	// should write header before writing the body,
+	// or client will receive HTTP 200 regardless the real status code
+	w.WriteHeader(statusCode)
+	w.Write(rBody)
+}
+
+func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath string) (string, error) {
+	cmd := exec.Command(command)
+
+	fi, err := os.Stat(srcPkgPath)
+	if err != nil {
+		return "", fmt.Errorf("could not find srcPkgPath: '%s'", srcPkgPath)
+	}
+	if fi.IsDir() {
+		cmd.Dir = srcPkgPath
+	} else {
+		cmd.Dir = path.Dir(srcPkgPath)
+	}
+
+	// set env variables for build command
+	cmd.Env = append(os.Environ(),
+		fmt.Sprintf("%v=%v", envSrcPkg, srcPkgPath),
+		fmt.Sprintf("%v=%v", envDeployPkg, deployPkgPath),
+	)
+
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", errors.Wrap(err, "error creating stdout pipe for cmd")
+	}
+
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return "", errors.Wrap(err, "error creating stderr pipe for cmd")
+	}
+
+	var buildLogs string
+
+	fmt.Printf("\n=== Build Logs ===")
+	// Init logs
+	fmt.Printf("command=%v\n", command)
+	fmt.Printf("env=%v\n", cmd.Env)
+
+	out := io.MultiReader(stdout, stderr)
+	scanner := bufio.NewScanner(out)
+
+	err = cmd.Start()
+	if err != nil {
+		return "", errors.Wrap(err, "error starting cmd")
+	}
+
+	// Runtime logs
+	for scanner.Scan() {
+		output := scanner.Text()
+		fmt.Println(output)
+		buildLogs += fmt.Sprintf("%v\n", output)
+	}
+
+	if err := scanner.Err(); err != nil {
+		scanErr := errors.Wrap(err, "error reading cmd output")
+		fmt.Println(scanErr)
+		return buildLogs, scanErr
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		cmdErr := errors.Wrapf(err, "error waiting for cmd %q", command)
+		fmt.Println(cmdErr)
+		return buildLogs, cmdErr
+	}
+	fmt.Printf("==================\n")
+
+	return buildLogs, nil
+}

pkg/builder/client/client.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+
+	builder "github.com/fission/fission/pkg/builder"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+type (
+	Client struct {
+		logger *zap.Logger
+		url    string
+	}
+)
+
+func MakeClient(logger *zap.Logger, builderUrl string) *Client {
+	return &Client{
+		logger: logger.Named("builder_client"),
+		url:    strings.TrimSuffix(builderUrl, "/"),
+	}
+}
+
+func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildResponse, error) {
+	body, err := json.Marshal(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "error marshaling json")
+	}
+
+	maxRetries := 20
+	var resp *http.Response
+
+	for i := 0; i < maxRetries; i++ {
+		resp, err = http.Post(c.url, "application/json", bytes.NewReader(body))
+
+		if err == nil {
+			if resp.StatusCode == 200 {
+				break
+			}
+			err = ferror.MakeErrorFromHTTP(resp)
+		}
+
+		if i < maxRetries-1 {
+			time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
+			c.logger.Error("error building package, retrying", zap.Error(err))
+			continue
+		}
+
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	rBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		c.logger.Error("error reading resp body", zap.Error(err))
+		return nil, err
+	}
+
+	pkgBuildResp := builder.PackageBuildResponse{}
+	err = json.Unmarshal([]byte(rBody), &pkgBuildResp)
+	if err != nil {
+		c.logger.Error("error parsing resp body", zap.Error(err))
+		return nil, err
+	}
+
+	return &pkgBuildResp, ferror.MakeErrorFromHTTP(resp)
+}

pkg/buildermgr/buildermgr.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/crd"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+)
+
+// Start the buildermgr service.
+func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
+	bmLogger := logger.Named("builder_manager")
+
+	fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to get fission or kubernetes client")
+	}
+
+	err = fissionClient.WaitForCRDs()
+	if err != nil {
+		return errors.Wrap(err, "error waiting for CRDs")
+	}
+
+	fetcherConfig, err := fetcherConfig.MakeFetcherConfig("/packages")
+	if err != nil {
+		return errors.Wrap(err, "error making fetcher config")
+	}
+
+	envWatcher := makeEnvironmentWatcher(bmLogger, fissionClient, kubernetesClient, fetcherConfig, envBuilderNamespace)
+	go envWatcher.watchEnvironments()
+
+	pkgWatcher := makePackageWatcher(bmLogger, fissionClient,
+		kubernetesClient, envBuilderNamespace, storageSvcUrl)
+	go pkgWatcher.watchPackages(fissionClient, kubernetesClient, envBuilderNamespace)
+
+	select {}
+}

pkg/buildermgr/common.go

@@ -0,0 +1,149 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/dchest/uniuri"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/builder"
+	builderClient "github.com/fission/fission/pkg/builder/client"
+	"github.com/fission/fission/pkg/crd"
+	ferror "github.com/fission/fission/pkg/error"
+	fetcherClient "github.com/fission/fission/pkg/fetcher/client"
+	"github.com/fission/fission/pkg/types"
+)
+
+// buildPackage helps to build source package into deployment package.
+// Following is the steps buildPackage function takes to complete the whole process.
+// 1. Send fetch request to fetcher to fetch source package.
+// 2. Send build request to builder to start a build.
+// 3. Send upload request to fetcher to upload deployment package.
+// 4. Return upload response and build logs.
+// *. Return build logs and error if any one of steps above failed.
+func buildPackage(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, envBuilderNamespace string,
+	storageSvcUrl string, pkg *fv1.Package) (uploadResp *types.ArchiveUploadResponse, buildLogs string, err error) {
+
+	env, err := fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
+	if err != nil {
+		e := "error getting environment CRD info"
+		logger.Error(e, zap.Error(err))
+		e = fmt.Sprintf("%s: %v", e, err)
+		return nil, e, ferror.MakeError(http.StatusInternalServerError, e)
+	}
+
+	svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, envBuilderNamespace)
+	srcPkgFilename := fmt.Sprintf("%v-%v", pkg.Metadata.Name, strings.ToLower(uniuri.NewLen(6)))
+	fetcherC := fetcherClient.MakeClient(logger, fmt.Sprintf("http://%v:8000", svcName))
+	builderC := builderClient.MakeClient(logger, fmt.Sprintf("http://%v:8001", svcName))
+
+	fetchReq := &types.FunctionFetchRequest{
+		FetchType:   types.FETCH_SOURCE,
+		Package:     pkg.Metadata,
+		Filename:    srcPkgFilename,
+		KeepArchive: false,
+	}
+
+	// send fetch request to fetcher
+	err = fetcherC.Fetch(ctx, fetchReq)
+	if err != nil {
+		e := "error fetching source package"
+		logger.Error(e, zap.Error(err))
+		e = fmt.Sprintf("%s: %v", e, err)
+		return nil, e, ferror.MakeError(http.StatusInternalServerError, e)
+	}
+
+	buildCmd := pkg.Spec.BuildCommand
+	if len(buildCmd) == 0 {
+		buildCmd = env.Spec.Builder.Command
+	}
+
+	pkgBuildReq := &builder.PackageBuildRequest{
+		SrcPkgFilename: srcPkgFilename,
+		BuildCommand:   buildCmd,
+	}
+
+	logger.Info("started building with source package", zap.String("source_package", srcPkgFilename))
+	// send build request to builder
+	buildResp, err := builderC.Build(pkgBuildReq)
+	if err != nil {
+		e := fmt.Sprintf("Error building deployment package: %v", err)
+		var buildLogs string
+		if buildResp != nil {
+			buildLogs = buildResp.BuildLogs
+		}
+		buildLogs += fmt.Sprintf("%v\n", e)
+		return nil, buildLogs, ferror.MakeError(http.StatusInternalServerError, e)
+	}
+
+	logger.Info("build succeed", zap.String("source_package", srcPkgFilename), zap.String("deployment_package", buildResp.ArtifactFilename))
+
+	archivePackage := !env.Spec.KeepArchive
+
+	uploadReq := &types.ArchiveUploadRequest{
+		Filename:       buildResp.ArtifactFilename,
+		StorageSvcUrl:  storageSvcUrl,
+		ArchivePackage: archivePackage,
+	}
+
+	logger.Info("started uploading deployment package", zap.String("deployment_package", buildResp.ArtifactFilename))
+	// ask fetcher to upload the deployment package
+	uploadResp, err = fetcherC.Upload(ctx, uploadReq)
+	if err != nil {
+		e := fmt.Sprintf("Error uploading deployment package: %v", err)
+		buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
+		return nil, buildResp.BuildLogs, ferror.MakeError(http.StatusInternalServerError, e)
+	}
+
+	return uploadResp, buildResp.BuildLogs, nil
+}
+
+func updatePackage(logger *zap.Logger, fissionClient *crd.FissionClient,
+	pkg *fv1.Package, status fv1.BuildStatus, buildLogs string,
+	uploadResp *types.ArchiveUploadResponse) (*fv1.Package, error) {
+
+	pkg.Status = fv1.PackageStatus{
+		BuildStatus: status,
+		BuildLog:    buildLogs,
+	}
+
+	if uploadResp != nil {
+		pkg.Spec.Deployment = fv1.Archive{
+			Type:     types.ArchiveTypeUrl,
+			URL:      uploadResp.ArchiveDownloadUrl,
+			Checksum: uploadResp.Checksum,
+		}
+	}
+
+	// update package spec
+	pkg, err := fissionClient.Packages(pkg.Metadata.Namespace).Update(pkg)
+	if err != nil {
+		e := "error updating package"
+		logger.Error(e, zap.Error(err))
+		return nil, errors.Wrap(err, e)
+	}
+
+	// return resource version for function to update function package ref
+	return pkg, nil
+}

pkg/buildermgr/envwatcher.go

@@ -0,0 +1,560 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/executor/util"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+)
+
+type requestType int
+
+const (
+	GET_BUILDER requestType = iota
+	CLEANUP_BUILDERS
+
+	LABEL_ENV_NAME            = "envName"
+	LABEL_ENV_NAMESPACE       = "envNamespace"
+	LABEL_ENV_RESOURCEVERSION = "envResourceVersion"
+	LABEL_DEPLOYMENT_OWNER    = "owner"
+	BUILDER_MGR               = "buildermgr"
+)
+
+var (
+	deletePropagation = metav1.DeletePropagationBackground
+	delOpt            = metav1.DeleteOptions{PropagationPolicy: &deletePropagation}
+)
+
+type (
+	builderInfo struct {
+		envMetadata *metav1.ObjectMeta
+		deployment  *v1beta1.Deployment
+		service     *apiv1.Service
+	}
+
+	envwRequest struct {
+		requestType
+		env      *fv1.Environment
+		envList  []fv1.Environment
+		respChan chan envwResponse
+	}
+
+	envwResponse struct {
+		builderInfo *builderInfo
+		err         error
+	}
+
+	environmentWatcher struct {
+		logger                 *zap.Logger
+		cache                  map[string]*builderInfo
+		requestChan            chan envwRequest
+		builderNamespace       string
+		fissionClient          *crd.FissionClient
+		kubernetesClient       *kubernetes.Clientset
+		fetcherConfig          *fetcherConfig.Config
+		builderImagePullPolicy apiv1.PullPolicy
+		useIstio               bool
+		collectorEndpoint      string
+	}
+)
+
+func makeEnvironmentWatcher(
+	logger *zap.Logger,
+	fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset,
+	fetcherConfig *fetcherConfig.Config,
+	builderNamespace string) *environmentWatcher {
+
+	useIstio := false
+	enableIstio := os.Getenv("ENABLE_ISTIO")
+	if len(enableIstio) > 0 {
+		istio, err := strconv.ParseBool(enableIstio)
+		if err != nil {
+			logger.Info("Failed to parse ENABLE_ISTIO, defaults to false")
+		}
+		useIstio = istio
+	}
+
+	builderImagePullPolicy := utils.GetImagePullPolicy(os.Getenv("BUILDER_IMAGE_PULL_POLICY"))
+
+	envWatcher := &environmentWatcher{
+		logger:                 logger.Named("environment_watcher"),
+		cache:                  make(map[string]*builderInfo),
+		requestChan:            make(chan envwRequest),
+		builderNamespace:       builderNamespace,
+		fissionClient:          fissionClient,
+		kubernetesClient:       kubernetesClient,
+		builderImagePullPolicy: builderImagePullPolicy,
+		useIstio:               useIstio,
+		fetcherConfig:          fetcherConfig,
+	}
+
+	go envWatcher.service()
+
+	return envWatcher
+}
+
+func (envw *environmentWatcher) getCacheKey(envName string, envNamespace string, envResourceVersion string) string {
+	return fmt.Sprintf("%v-%v-%v", envName, envNamespace, envResourceVersion)
+}
+
+func (env *environmentWatcher) getLabelForDeploymentOwner() map[string]string {
+	return map[string]string{
+		LABEL_DEPLOYMENT_OWNER: BUILDER_MGR,
+	}
+}
+
+func (envw *environmentWatcher) getLabels(envName string, envNamespace string, envResourceVersion string) map[string]string {
+	return map[string]string{
+		LABEL_ENV_NAME:            envName,
+		LABEL_ENV_NAMESPACE:       envNamespace,
+		LABEL_ENV_RESOURCEVERSION: envResourceVersion,
+		LABEL_DEPLOYMENT_OWNER:    BUILDER_MGR,
+	}
+}
+
+func (envw *environmentWatcher) watchEnvironments() {
+	rv := ""
+	for {
+		wi, err := envw.fissionClient.Environments(metav1.NamespaceAll).Watch(metav1.ListOptions{
+			ResourceVersion: rv,
+		})
+		if err != nil {
+			if utils.IsNetworkError(err) {
+				envw.logger.Error("encountered network error, retrying later", zap.Error(err))
+				time.Sleep(5 * time.Second)
+				continue
+			}
+			envw.logger.Fatal("error watching environment list", zap.Error(err))
+		}
+
+		for {
+			ev, more := <-wi.ResultChan()
+			if !more {
+				// restart watch from last rv
+				break
+			}
+			if ev.Type == watch.Error {
+				// restart watch from the start
+				rv = ""
+				time.Sleep(time.Second)
+				break
+			}
+			env := ev.Object.(*fv1.Environment)
+			rv = env.Metadata.ResourceVersion
+			envw.sync()
+		}
+	}
+}
+
+func (envw *environmentWatcher) sync() {
+	maxRetries := 10
+	for i := 0; i < maxRetries; i++ {
+		envList, err := envw.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
+		if err != nil {
+			if utils.IsNetworkError(err) {
+				envw.logger.Error("error syncing environment CRD resources due to network error, retrying later", zap.Error(err))
+				time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
+				continue
+			}
+			envw.logger.Fatal("error syncing environment CRD resources", zap.Error(err))
+		}
+
+		// Create environment builders for all environments
+		for i := range envList.Items {
+			env := envList.Items[i]
+
+			if env.Spec.Version == 1 || // builder is not supported with v1 interface
+				len(env.Spec.Builder.Image) == 0 { // ignore env without builder image
+				continue
+			}
+			_, err := envw.getEnvBuilder(&env)
+			if err != nil {
+				envw.logger.Error("error creating builder", zap.Error(err), zap.String("builder_target", env.Metadata.Name))
+			}
+		}
+
+		// Remove environment builders no longer needed
+		envw.cleanupEnvBuilders(envList.Items)
+		break
+	}
+}
+
+func (envw *environmentWatcher) service() {
+	for {
+		req := <-envw.requestChan
+		switch req.requestType {
+		case GET_BUILDER:
+			// In order to support backward compatibility, for all environments with builder image created in default env,
+			// the pods will be created in fission-builder namespace
+			ns := envw.builderNamespace
+			if req.env.Metadata.Namespace != metav1.NamespaceDefault {
+				ns = req.env.Metadata.Namespace
+			}
+
+			key := envw.getCacheKey(req.env.Metadata.Name, ns, req.env.Metadata.ResourceVersion)
+			builderInfo, ok := envw.cache[key]
+			if !ok {
+				builderInfo, err := envw.createBuilder(req.env, ns)
+				if err != nil {
+					req.respChan <- envwResponse{err: err}
+					continue
+				}
+				envw.cache[key] = builderInfo
+			}
+			req.respChan <- envwResponse{builderInfo: builderInfo}
+
+		case CLEANUP_BUILDERS:
+			latestEnvList := make(map[string]*fv1.Environment)
+			for i := range req.envList {
+				env := req.envList[i]
+				// In order to support backward compatibility, for all builder images created in default
+				// env, the pods are created in fission-builder namespace
+				ns := envw.builderNamespace
+				if env.Metadata.Namespace != metav1.NamespaceDefault {
+					ns = env.Metadata.Namespace
+				}
+				key := envw.getCacheKey(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
+				latestEnvList[key] = &env
+			}
+
+			// If an environment is deleted when builder manager down,
+			// the builder belongs to the environment will be out-of-
+			// control (an orphan builder) since there is no record in
+			// cache and CRD. We need to iterate over the services &
+			// deployments to remove both normal and orphan builders.
+
+			svcList, err := envw.getBuilderServiceList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
+			if err != nil {
+				envw.logger.Error("error getting the builder service list", zap.Error(err))
+			}
+			for _, svc := range svcList {
+				envName := svc.ObjectMeta.Labels[LABEL_ENV_NAME]
+				envNamespace := svc.ObjectMeta.Labels[LABEL_ENV_NAMESPACE]
+				envResourceVersion := svc.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION]
+				key := envw.getCacheKey(envName, envNamespace, envResourceVersion)
+				if _, ok := latestEnvList[key]; !ok {
+					err := envw.deleteBuilderServiceByName(svc.ObjectMeta.Name, svc.ObjectMeta.Namespace)
+					if err != nil {
+						envw.logger.Error("error removing builder service", zap.Error(err),
+							zap.String("service_name", svc.ObjectMeta.Name),
+							zap.String("service_namespace", svc.ObjectMeta.Namespace))
+					}
+				}
+				delete(envw.cache, key)
+			}
+
+			deployList, err := envw.getBuilderDeploymentList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
+			if err != nil {
+				envw.logger.Error("error getting the builder deployment list", zap.Error(err))
+			}
+			for _, deploy := range deployList {
+				envName := deploy.ObjectMeta.Labels[LABEL_ENV_NAME]
+				envNamespace := deploy.ObjectMeta.Labels[LABEL_ENV_NAMESPACE]
+				envResourceVersion := deploy.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION]
+				key := envw.getCacheKey(envName, envNamespace, envResourceVersion)
+				if _, ok := latestEnvList[key]; !ok {
+					err := envw.deleteBuilderDeploymentByName(deploy.ObjectMeta.Name, deploy.ObjectMeta.Namespace)
+					if err != nil {
+						envw.logger.Error("error removing builder deployment", zap.Error(err),
+							zap.String("deployment_name", deploy.ObjectMeta.Name),
+							zap.String("deployment_namespace", deploy.ObjectMeta.Namespace))
+					}
+				}
+				delete(envw.cache, key)
+			}
+		}
+	}
+}
+
+func (envw *environmentWatcher) getEnvBuilder(env *fv1.Environment) (*builderInfo, error) {
+	respChan := make(chan envwResponse)
+	envw.requestChan <- envwRequest{
+		requestType: GET_BUILDER,
+		env:         env,
+		respChan:    respChan,
+	}
+	resp := <-respChan
+	return resp.builderInfo, resp.err
+}
+
+func (envw *environmentWatcher) cleanupEnvBuilders(envs []fv1.Environment) {
+	envw.requestChan <- envwRequest{
+		requestType: CLEANUP_BUILDERS,
+		envList:     envs,
+	}
+}
+
+func (envw *environmentWatcher) createBuilder(env *fv1.Environment, ns string) (*builderInfo, error) {
+	var svc *apiv1.Service
+	var deploy *v1beta1.Deployment
+
+	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
+
+	svcList, err := envw.getBuilderServiceList(sel, ns)
+	if err != nil {
+		return nil, err
+	}
+	// there should be only one service in svcList
+	if len(svcList) == 0 {
+		svc, err = envw.createBuilderService(env, ns)
+		if err != nil {
+			return nil, errors.Wrap(err, "error creating builder service")
+		}
+	} else if len(svcList) == 1 {
+		svc = &svcList[0]
+	} else {
+		return nil, fmt.Errorf("found more than one builder service for environment %q", env.Metadata.Name)
+	}
+
+	deployList, err := envw.getBuilderDeploymentList(sel, ns)
+	if err != nil {
+		return nil, err
+	}
+	// there should be only one deploy in deployList
+	if len(deployList) == 0 {
+		// create builder SA in this ns, if not already created
+		_, err := utils.SetupSA(envw.kubernetesClient, types.FissionBuilderSA, ns)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error creating %q in ns: %s", types.FissionBuilderSA, ns)
+		}
+
+		deploy, err = envw.createBuilderDeployment(env, ns)
+		if err != nil {
+			return nil, errors.Wrap(err, "error creating builder deployment")
+		}
+	} else if len(deployList) == 1 {
+		deploy = &deployList[0]
+	} else {
+		return nil, fmt.Errorf("found more than one builder deployment for environment %q", env.Metadata.Name)
+	}
+
+	return &builderInfo{
+		envMetadata: &env.Metadata,
+		service:     svc,
+		deployment:  deploy,
+	}, nil
+}
+
+func (envw *environmentWatcher) deleteBuilderServiceByName(name, namespace string) error {
+	err := envw.kubernetesClient.CoreV1().
+		Services(namespace).
+		Delete(name, &delOpt)
+	if err != nil {
+		return errors.Wrapf(err, "error deleting builder service %s.%s", name, namespace)
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) deleteBuilderDeploymentByName(name, namespace string) error {
+	err := envw.kubernetesClient.ExtensionsV1beta1().
+		Deployments(namespace).
+		Delete(name, &delOpt)
+	if err != nil {
+		return errors.Wrapf(err, "error deleting builder deployment %s.%s", name, namespace)
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) deleteBuilderService(sel map[string]string, ns string) error {
+	svcList, err := envw.getBuilderServiceList(sel, ns)
+	if err != nil {
+		return err
+	}
+	for _, svc := range svcList {
+		envw.logger.Info("removing builder service", zap.String("service_name", svc.ObjectMeta.Name))
+		err = envw.kubernetesClient.CoreV1().
+			Services(ns).
+			Delete(svc.ObjectMeta.Name, &delOpt)
+		if err != nil {
+			return errors.Wrap(err, "error deleting builder service")
+		}
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) deleteBuilderDeployment(sel map[string]string, ns string) error {
+	deployList, err := envw.getBuilderDeploymentList(sel, ns)
+	if err != nil {
+		return err
+	}
+	for _, deploy := range deployList {
+		envw.logger.Info("removing builder deployment", zap.String("deployment_name", deploy.ObjectMeta.Name))
+		err = envw.kubernetesClient.ExtensionsV1beta1().
+			Deployments(ns).
+			Delete(deploy.ObjectMeta.Name, &delOpt)
+		if err != nil {
+			return errors.Wrap(err, "error deleteing builder deployment")
+		}
+	}
+	return nil
+}
+
+func (envw *environmentWatcher) getBuilderServiceList(sel map[string]string, ns string) ([]apiv1.Service, error) {
+	svcList, err := envw.kubernetesClient.CoreV1().Services(ns).List(
+		metav1.ListOptions{
+			LabelSelector: labels.Set(sel).AsSelector().String(),
+		})
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting builder service list")
+	}
+	return svcList.Items, nil
+}
+
+func (envw *environmentWatcher) createBuilderService(env *fv1.Environment, ns string) (*apiv1.Service, error) {
+	name := fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion)
+	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
+	service := apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns,
+			Name:      name,
+			Labels:    sel,
+		},
+		Spec: apiv1.ServiceSpec{
+			Selector: sel,
+			Type:     apiv1.ServiceTypeClusterIP,
+			Ports: []apiv1.ServicePort{
+				{
+					Name:     "fetcher-port",
+					Protocol: apiv1.ProtocolTCP,
+					Port:     8000,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 8000,
+					},
+				},
+				{
+					Name:     "builder-port",
+					Protocol: apiv1.ProtocolTCP,
+					Port:     8001,
+					TargetPort: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 8001,
+					},
+				},
+			},
+		},
+	}
+	envw.logger.Info("creating builder service", zap.String("service_name", name))
+	_, err := envw.kubernetesClient.CoreV1().Services(ns).Create(&service)
+	if err != nil {
+		return nil, err
+	}
+	return &service, nil
+}
+
+func (envw *environmentWatcher) getBuilderDeploymentList(sel map[string]string, ns string) ([]v1beta1.Deployment, error) {
+	deployList, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).List(
+		metav1.ListOptions{
+			LabelSelector: labels.Set(sel).AsSelector().String(),
+		})
+	if err != nil {
+		return nil, errors.Wrap(err, "error getting builder deployment list")
+	}
+	return deployList.Items, nil
+}
+
+func (envw *environmentWatcher) createBuilderDeployment(env *fv1.Environment, ns string) (*v1beta1.Deployment, error) {
+	name := fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion)
+	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
+	var replicas int32 = 1
+
+	podAnnotations := env.Metadata.Annotations
+	if podAnnotations == nil {
+		podAnnotations = make(map[string]string)
+	}
+	if envw.useIstio && env.Spec.AllowAccessToExternalNetwork {
+		podAnnotations["sidecar.istio.io/inject"] = "false"
+	}
+
+	deployment := &v1beta1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns,
+			Name:      name,
+			Labels:    sel,
+		},
+		Spec: v1beta1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: sel,
+			},
+			Template: apiv1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      sel,
+					Annotations: podAnnotations,
+				},
+				Spec: apiv1.PodSpec{
+					Containers: []apiv1.Container{
+						util.MergeContainerSpecs(&apiv1.Container{
+							Name:                   "builder",
+							Image:                  env.Spec.Builder.Image,
+							ImagePullPolicy:        envw.builderImagePullPolicy,
+							TerminationMessagePath: "/dev/termination-log",
+							Command:                []string{"/builder", envw.fetcherConfig.SharedMountPath()},
+							ReadinessProbe: &apiv1.Probe{
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       2,
+								Handler: apiv1.Handler{
+									HTTPGet: &apiv1.HTTPGetAction{
+										Path: "/healthz",
+										Port: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: 8001,
+										},
+									},
+								},
+							},
+						}, env.Spec.Builder.Container),
+					},
+					ServiceAccountName: "fission-builder",
+				},
+			},
+		},
+	}
+
+	err := envw.fetcherConfig.AddFetcherToPodSpec(&deployment.Spec.Template.Spec, "builder")
+	if err != nil {
+		return nil, err
+	}
+
+	envw.logger.Info("creating builder deployment", zap.String("deployment", name))
+	_, err = envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Create(deployment)
+	if err != nil {
+		return nil, err
+	}
+
+	return deployment, nil
+}

pkg/buildermgr/pkgwatcher.go

@@ -0,0 +1,250 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package buildermgr
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/cache"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+)
+
+type (
+	packageWatcher struct {
+		logger           *zap.Logger
+		fissionClient    *crd.FissionClient
+		k8sClient        *kubernetes.Clientset
+		podStore         k8sCache.Store
+		pkgStore         k8sCache.Store
+		builderNamespace string
+		storageSvcUrl    string
+	}
+)
+
+func makePackageWatcher(logger *zap.Logger, fissionClient *crd.FissionClient, k8sClientSet *kubernetes.Clientset,
+	builderNamespace string, storageSvcUrl string) *packageWatcher {
+	lw := k8sCache.NewListWatchFromClient(k8sClientSet.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.Everything())
+	store, controller := k8sCache.NewInformer(lw, &apiv1.Pod{}, 30*time.Second, k8sCache.ResourceEventHandlerFuncs{})
+	go controller.Run(make(chan struct{}))
+
+	pkgw := &packageWatcher{
+		logger:           logger.Named("package_watcher"),
+		fissionClient:    fissionClient,
+		k8sClient:        k8sClientSet,
+		podStore:         store,
+		builderNamespace: builderNamespace,
+		storageSvcUrl:    storageSvcUrl,
+	}
+	return pkgw
+}
+
+// build helps to update package status, checks environment builder pod status and
+// dispatches buildPackage to build source package into deployment package.
+// Following is the steps build function takes to complete the whole process.
+// 1. Check package status
+// 2. Update package status to running state
+// 3. Check environment builder pod status
+// 4. Call buildPackage to build package
+// 5. Update package resource in package ref of functions that share the same package
+// 6. Update package status to succeed state
+// *. Update package status to failed state,if any one of steps above failed/time out
+func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *fv1.Package) {
+
+	// Ignore non-pending state packages.
+	if srcpkg.Status.BuildStatus != fv1.BuildStatusPending {
+		return
+	}
+
+	// Ignore duplicate build requests
+	key := fmt.Sprintf("%v-%v", srcpkg.Metadata.Name, srcpkg.Metadata.ResourceVersion)
+	err, _ := buildCache.Set(key, srcpkg)
+	if err != nil {
+		return
+	}
+	defer buildCache.Delete(key)
+
+	pkgw.logger.Info("starting build for package", zap.String("package_name", srcpkg.Metadata.Name), zap.String("resource_version", srcpkg.Metadata.ResourceVersion))
+
+	pkg, err := updatePackage(pkgw.logger, pkgw.fissionClient, srcpkg, fv1.BuildStatusRunning, "", nil)
+	if err != nil {
+		pkgw.logger.Error("error setting package pending state", zap.Error(err))
+		return
+	}
+
+	env, err := pkgw.fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
+	if k8serrors.IsNotFound(err) {
+		e := "environment does not exist"
+		pkgw.logger.Error(e, zap.String("environment", pkg.Spec.Environment.Name))
+		updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
+			fv1.BuildStatusFailed, fmt.Sprintf("%s: %q", e, pkg.Spec.Environment.Name), nil)
+		return
+	}
+
+	// Do health check for environment builder pod
+	for i := 0; i < 15; i++ {
+		// Informer store is not able to use label to find the pod,
+		// iterate all available environment builders.
+		items := pkgw.podStore.List()
+		if err != nil {
+			pkgw.logger.Error("error retrieving pod information for environment", zap.Error(err), zap.String("environment", env.Metadata.Name))
+			return
+		}
+
+		if len(items) == 0 {
+			pkgw.logger.Info("builder pod does not exist for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name))
+			time.Sleep(time.Duration(i*1) * time.Second)
+			continue
+		}
+
+		for _, item := range items {
+			pod := item.(*apiv1.Pod)
+
+			// In order to support backward compatibility, for all builder images created in default env,
+			// the pods will be created in fission-builder namespace
+			builderNs := pkgw.builderNamespace
+			if env.Metadata.Namespace != metav1.NamespaceDefault {
+				builderNs = env.Metadata.Namespace
+			}
+
+			// Filter non-matching pods
+			if pod.ObjectMeta.Labels[LABEL_ENV_NAME] != env.Metadata.Name ||
+				pod.ObjectMeta.Labels[LABEL_ENV_NAMESPACE] != builderNs ||
+				pod.ObjectMeta.Labels[LABEL_ENV_RESOURCEVERSION] != env.Metadata.ResourceVersion {
+				continue
+			}
+
+			// Pod may become "Running" state but still failed at health check, so use
+			// pod.Status.ContainerStatuses instead of pod.Status.Phase to check pod readiness states.
+			podIsReady := true
+
+			for _, cStatus := range pod.Status.ContainerStatuses {
+				podIsReady = podIsReady && cStatus.Ready
+			}
+
+			if !podIsReady {
+				pkgw.logger.Info("builder pod is not ready for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name))
+				time.Sleep(time.Duration(i*1) * time.Second)
+				break
+			}
+
+			// Add the package getter rolebinding to builder sa
+			// we continue here if role binding was not setup succeesffully. this is because without this, the fetcher wont be able to fetch the source pkg into the container and
+			// the build will fail eventually
+			err := utils.SetupRoleBinding(pkgw.logger, pkgw.k8sClient, types.PackageGetterRB, pkg.Metadata.Namespace, types.PackageGetterCR, types.ClusterRole, types.FissionBuilderSA, builderNs)
+			if err != nil {
+				pkgw.logger.Error("error setting up role binding for package",
+					zap.Error(err),
+					zap.String("role_binding", types.PackageGetterRB),
+					zap.String("package_name", pkg.Metadata.Name),
+					zap.String("package_namespace", pkg.Metadata.Namespace))
+				continue
+			} else {
+				pkgw.logger.Info("setup rolebinding for sa package",
+					zap.String("sa", fmt.Sprintf("%s.%s", types.FissionBuilderSA, builderNs)),
+					zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)))
+			}
+
+			ctx := context.Background()
+			uploadResp, buildLogs, err := buildPackage(ctx, pkgw.logger, pkgw.fissionClient, builderNs, pkgw.storageSvcUrl, pkg)
+			if err != nil {
+				pkgw.logger.Error("error building package", zap.Error(err), zap.String("package_name", pkg.Metadata.Name))
+				updatePackage(pkgw.logger, pkgw.fissionClient, pkg, types.BuildStatusFailed, buildLogs, nil)
+				return
+			}
+
+			pkgw.logger.Info("starting package info update", zap.String("package_name", pkg.Metadata.Name))
+
+			fnList, err := pkgw.fissionClient.
+				Functions(metav1.NamespaceAll).List(metav1.ListOptions{})
+			if err != nil {
+				e := "error getting function list"
+				pkgw.logger.Error(e, zap.Error(err))
+				buildLogs += fmt.Sprintf("%s: %v\n", e, err)
+				updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fv1.BuildStatusFailed, buildLogs, nil)
+			}
+
+			// A package may be used by multiple functions. Update
+			// functions with old package resource version
+			for _, fn := range fnList.Items {
+				if fn.Spec.Package.PackageRef.Name == pkg.Metadata.Name &&
+					fn.Spec.Package.PackageRef.Namespace == pkg.Metadata.Namespace &&
+					fn.Spec.Package.PackageRef.ResourceVersion != pkg.Metadata.ResourceVersion {
+					fn.Spec.Package.PackageRef.ResourceVersion = pkg.Metadata.ResourceVersion
+					// update CRD
+					_, err = pkgw.fissionClient.Functions(fn.Metadata.Namespace).Update(&fn)
+					if err != nil {
+						e := "error updating function package resource version"
+						pkgw.logger.Error(e, zap.Error(err))
+						buildLogs += fmt.Sprintf("%s: %v\n", e, err)
+						updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fv1.BuildStatusFailed, buildLogs, nil)
+						return
+					}
+				}
+			}
+
+			_, err = updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
+				types.BuildStatusSucceeded, buildLogs, uploadResp)
+			if err != nil {
+				pkgw.logger.Error("error updating package info", zap.Error(err), zap.String("package_name", pkg.Metadata.Name))
+				updatePackage(pkgw.logger, pkgw.fissionClient, pkg, types.BuildStatusFailed, buildLogs, nil)
+				return
+			}
+
+			pkgw.logger.Info("completed package build request", zap.String("package_name", pkg.Metadata.Name))
+			return
+		}
+	}
+	// build timeout
+	updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
+		types.BuildStatusFailed, "Build timeout due to environment builder not ready", nil)
+
+	pkgw.logger.Error("max retries exceeded in building source package, timeout due to environment builder not ready",
+		zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)))
+
+	return
+}
+
+func (pkgw *packageWatcher) watchPackages(fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset, builderNamespace string) {
+	buildCache := cache.MakeCache(0, 0)
+	lw := k8sCache.NewListWatchFromClient(pkgw.fissionClient.GetCrdClient(), "packages", apiv1.NamespaceAll, fields.Everything())
+	pkgStore, controller := k8sCache.NewInformer(lw, &fv1.Package{}, 60*time.Second, k8sCache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			pkg := obj.(*fv1.Package)
+			go pkgw.build(buildCache, pkg)
+		},
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			pkg := newObj.(*fv1.Package)
+			go pkgw.build(buildCache, pkg)
+		},
+	})
+	pkgw.pkgStore = pkgStore
+	controller.Run(make(chan struct{}))
+}

pkg/cache/cache.go

@@ -0,0 +1,202 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"fmt"
+	"time"
+
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+type requestType int
+
+const (
+	GET requestType = iota
+	SET
+	DELETE
+	EXPIRE
+	COPY
+)
+
+type (
+	Value struct {
+		ctime time.Time
+		atime time.Time
+		value interface{}
+	}
+	Cache struct {
+		cache          map[interface{}]*Value
+		ctimeExpiry    time.Duration
+		atimeExpiry    time.Duration
+		requestChannel chan *request
+	}
+
+	request struct {
+		requestType
+		key             interface{}
+		value           interface{}
+		responseChannel chan *response
+	}
+	response struct {
+		error
+		existingValue interface{}
+		mapCopy       map[interface{}]interface{}
+		value         interface{}
+	}
+)
+
+func (c *Cache) IsOld(v *Value) bool {
+	if (c.ctimeExpiry != time.Duration(0)) && (time.Since(v.ctime) > c.ctimeExpiry) {
+		return true
+	}
+
+	if (c.atimeExpiry != time.Duration(0)) && (time.Since(v.atime) > c.atimeExpiry) {
+		return true
+	}
+
+	return false
+}
+
+func MakeCache(ctimeExpiry, atimeExpiry time.Duration) *Cache {
+	c := &Cache{
+		cache:          make(map[interface{}]*Value),
+		ctimeExpiry:    ctimeExpiry,
+		atimeExpiry:    atimeExpiry,
+		requestChannel: make(chan *request),
+	}
+	go c.service()
+	if ctimeExpiry != time.Duration(0) || atimeExpiry != time.Duration(0) {
+		go c.expiryService()
+	}
+	return c
+}
+
+func (c *Cache) service() {
+	for {
+		req := <-c.requestChannel
+		resp := &response{}
+		switch req.requestType {
+		case GET:
+			val, ok := c.cache[req.key]
+			if !ok {
+				resp.error = ferror.MakeError(ferror.ErrorNotFound,
+					fmt.Sprintf("key '%v' not found", req.key))
+			} else if c.IsOld(val) {
+				resp.error = ferror.MakeError(ferror.ErrorNotFound,
+					fmt.Sprintf("key '%v' expired (atime %v)", req.key, val.atime))
+				delete(c.cache, req.key)
+			} else {
+				// update atime
+				val.atime = time.Now()
+				c.cache[req.key] = val
+				resp.value = val.value
+			}
+			req.responseChannel <- resp
+		case SET:
+			now := time.Now()
+			if _, ok := c.cache[req.key]; ok {
+				val := c.cache[req.key]
+				val.atime = time.Now()
+				resp.existingValue = val.value
+				resp.error = ferror.MakeError(ferror.ErrorNameExists, "key already exists")
+			} else {
+				c.cache[req.key] = &Value{
+					value: req.value,
+					ctime: now,
+					atime: now,
+				}
+			}
+			req.responseChannel <- resp
+		case DELETE:
+			delete(c.cache, req.key)
+			req.responseChannel <- resp
+		case EXPIRE:
+			for k, v := range c.cache {
+				if c.IsOld(v) {
+					delete(c.cache, k)
+				}
+			}
+			// no response
+		case COPY:
+			resp.mapCopy = make(map[interface{}]interface{})
+			for k, v := range c.cache {
+				resp.mapCopy[k] = v.value
+			}
+			req.responseChannel <- resp
+		default:
+			resp.error = ferror.MakeError(ferror.ErrorInvalidArgument,
+				fmt.Sprintf("invalid request type: %v", req.requestType))
+			req.responseChannel <- resp
+		}
+	}
+}
+
+func (c *Cache) Get(key interface{}) (interface{}, error) {
+	respChannel := make(chan *response)
+	c.requestChannel <- &request{
+		requestType:     GET,
+		key:             key,
+		responseChannel: respChannel,
+	}
+	resp := <-respChannel
+	return resp.value, resp.error
+}
+
+// if key exists in the cache, the new value is NOT set; instead an
+// error and the old value are returned
+func (c *Cache) Set(key interface{}, value interface{}) (error, interface{}) {
+	respChannel := make(chan *response)
+	c.requestChannel <- &request{
+		requestType:     SET,
+		key:             key,
+		value:           value,
+		responseChannel: respChannel,
+	}
+	resp := <-respChannel
+	return resp.error, resp.existingValue
+}
+
+func (c *Cache) Delete(key interface{}) error {
+	respChannel := make(chan *response)
+	c.requestChannel <- &request{
+		requestType:     DELETE,
+		key:             key,
+		responseChannel: respChannel,
+	}
+	resp := <-respChannel
+	return resp.error
+}
+
+func (c *Cache) Copy() map[interface{}]interface{} {
+	respChannel := make(chan *response)
+	c.requestChannel <- &request{
+		requestType:     COPY,
+		responseChannel: respChannel,
+	}
+	resp := <-respChannel
+	return resp.mapCopy
+}
+
+func (c *Cache) expiryService() {
+	for {
+		time.Sleep(time.Minute)
+		c.requestChannel <- &request{
+			requestType: EXPIRE,
+		}
+	}
+}

pkg/cache/cache_test.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"log"
+	"testing"
+	"time"
+)
+
+func checkErr(err error) {
+	if err != nil {
+		log.Panicf("err: %v", err)
+	}
+}
+
+func TestCache(t *testing.T) {
+	c := MakeCache(100*time.Millisecond, 100*time.Millisecond)
+
+	err, _ := c.Set("a", "b")
+	checkErr(err)
+	err, _ = c.Set("p", "q")
+	checkErr(err)
+
+	val, err := c.Get("a")
+	checkErr(err)
+	if val != "b" {
+		log.Panicf("value %v", val)
+	}
+
+	cc := c.Copy()
+	if len(cc) != 2 {
+		log.Panicf("expected 2 items")
+	}
+
+	err = c.Delete("a")
+	checkErr(err)
+
+	_, err = c.Get("a")
+	if err == nil {
+		log.Panicf("found deleted element")
+	}
+
+	err, _ = c.Set("expires", "42")
+	checkErr(err)
+	time.Sleep(150 * time.Millisecond)
+	_, err = c.Get("expires")
+	if err == nil {
+		log.Panicf("found expired element")
+	}
+}

pkg/canaryconfigmgr/canaryConfigCache.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package canaryconfigmgr
+
+import (
+	"context"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission/pkg/cache"
+)
+
+type (
+	canaryConfigCancelFuncMap struct {
+		cache *cache.Cache // map[metadataKey]*context.Context
+	}
+
+	// metav1.ObjectMeta is not hashable, so we make a hashable copy
+	// of the subset of its fields that are identifiable.
+	metadataKey struct {
+		Name      string
+		Namespace string
+	}
+
+	CanaryProcessingInfo struct {
+		CancelFunc *context.CancelFunc
+		Ticker     *time.Ticker
+	}
+)
+
+func makecanaryConfigCancelFuncMap() *canaryConfigCancelFuncMap {
+	return &canaryConfigCancelFuncMap{
+		cache: cache.MakeCache(0, 0),
+	}
+}
+
+func keyFromMetadata(m *metav1.ObjectMeta) metadataKey {
+	return metadataKey{
+		Name:      m.Name,
+		Namespace: m.Namespace,
+	}
+}
+
+func (cancelFuncMap *canaryConfigCancelFuncMap) lookup(f *metav1.ObjectMeta) (*CanaryProcessingInfo, error) {
+	mk := keyFromMetadata(f)
+	item, err := cancelFuncMap.cache.Get(mk)
+	if err != nil {
+		return nil, err
+	}
+	value := item.(*CanaryProcessingInfo)
+	return value, nil
+}
+
+func (cancelFuncMap *canaryConfigCancelFuncMap) assign(f *metav1.ObjectMeta, value *CanaryProcessingInfo) error {
+	mk := keyFromMetadata(f)
+	err, _ := cancelFuncMap.cache.Set(mk, value)
+	return err
+}
+
+func (cancelFuncMap *canaryConfigCancelFuncMap) remove(f *metav1.ObjectMeta) error {
+	mk := keyFromMetadata(f)
+	return cancelFuncMap.cache.Delete(mk)
+}

pkg/canaryconfigmgr/canaryConfigMgr.go

@@ -0,0 +1,530 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package canaryconfigmgr
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/types"
+)
+
+const (
+	maxRetries = 10
+)
+
+type canaryConfigMgr struct {
+	logger                 *zap.Logger
+	fissionClient          *crd.FissionClient
+	kubeClient             *kubernetes.Clientset
+	canaryConfigStore      k8sCache.Store
+	canaryConfigController k8sCache.Controller
+	promClient             *PrometheusApiClient
+	crdClient              *rest.RESTClient
+	canaryCfgCancelFuncMap *canaryConfigCancelFuncMap
+}
+
+func MakeCanaryConfigMgr(logger *zap.Logger, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, crdClient *rest.RESTClient, prometheusSvc string) (*canaryConfigMgr, error) {
+	if prometheusSvc == "" {
+		logger.Info("try to retrieve prometheus server information from environment variables")
+
+		var prometheusSvcHost, prometheusSvcPort string
+		// handle a case where there is a prometheus server is already installed, try to find the service from env variable
+		envVars := os.Environ()
+
+		for _, envVar := range envVars {
+			if strings.Contains(envVar, "PROMETHEUS_SERVER_SERVICE_HOST") {
+				prometheusSvcHost = getEnvValue(envVar)
+			} else if strings.Contains(envVar, "PROMETHEUS_SERVER_SERVICE_PORT") {
+				prometheusSvcPort = getEnvValue(envVar)
+			}
+
+			if len(prometheusSvcHost) > 0 && len(prometheusSvcPort) > 0 {
+				break
+			}
+		}
+		prometheusSvc = fmt.Sprintf("http://%v:%v", prometheusSvcHost, prometheusSvcPort)
+	}
+
+	_, err := url.Parse(prometheusSvc)
+	if err != nil {
+		return nil, fmt.Errorf("prometheus service url not found/invalid, cant create canary config manager: %v", prometheusSvc)
+	}
+
+	promClient, err := MakePrometheusClient(logger, prometheusSvc)
+	if err != nil {
+		return nil, err
+	}
+
+	configMgr := &canaryConfigMgr{
+		logger:                 logger.Named("canary_config_manager"),
+		fissionClient:          fissionClient,
+		kubeClient:             kubeClient,
+		crdClient:              crdClient,
+		promClient:             promClient,
+		canaryCfgCancelFuncMap: makecanaryConfigCancelFuncMap(),
+	}
+
+	store, controller := configMgr.initCanaryConfigController()
+	configMgr.canaryConfigStore = store
+	configMgr.canaryConfigController = controller
+
+	return configMgr, nil
+}
+
+func (canaryCfgMgr *canaryConfigMgr) initCanaryConfigController() (k8sCache.Store, k8sCache.Controller) {
+	resyncPeriod := 30 * time.Second
+	listWatch := k8sCache.NewListWatchFromClient(canaryCfgMgr.crdClient, "canaryconfigs", metav1.NamespaceAll, fields.Everything())
+	store, controller := k8sCache.NewInformer(listWatch, &fv1.CanaryConfig{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				canaryConfig := obj.(*fv1.CanaryConfig)
+				if canaryConfig.Status.Status == types.CanaryConfigStatusPending {
+					go canaryCfgMgr.addCanaryConfig(canaryConfig)
+				}
+			},
+			DeleteFunc: func(obj interface{}) {
+				canaryConfig := obj.(*fv1.CanaryConfig)
+				go canaryCfgMgr.deleteCanaryConfig(canaryConfig)
+			},
+			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
+				oldConfig := oldObj.(*fv1.CanaryConfig)
+				newConfig := newObj.(*fv1.CanaryConfig)
+				if oldConfig.Metadata.ResourceVersion != newConfig.Metadata.ResourceVersion &&
+					newConfig.Status.Status == types.CanaryConfigStatusPending {
+					canaryCfgMgr.logger.Info("update canary config invoked",
+						zap.String("name", newConfig.Metadata.Name),
+						zap.String("namespace", newConfig.Metadata.Namespace),
+						zap.String("version", newConfig.Metadata.ResourceVersion))
+					go canaryCfgMgr.updateCanaryConfig(oldConfig, newConfig)
+				}
+				go canaryCfgMgr.reSyncCanaryConfigs()
+
+			},
+		})
+
+	return store, controller
+}
+
+func (canaryCfgMgr *canaryConfigMgr) Run(ctx context.Context) {
+	go canaryCfgMgr.canaryConfigController.Run(ctx.Done())
+	canaryCfgMgr.logger.Info("started canary configmgr controller")
+}
+
+func (canaryCfgMgr *canaryConfigMgr) addCanaryConfig(canaryConfig *fv1.CanaryConfig) {
+	canaryCfgMgr.logger.Info("addCanaryConfig called", zap.String("canary_config", canaryConfig.Metadata.Name))
+
+	// for each canary config, create a ticker with increment interval
+	interval, err := time.ParseDuration(canaryConfig.Spec.WeightIncrementDuration)
+	if err != nil {
+		canaryCfgMgr.logger.Error("error parsing duration - cant proceed with this canaryConfig",
+			zap.Error(err),
+			zap.String("duration", canaryConfig.Spec.WeightIncrementDuration),
+			zap.String("name", canaryConfig.Metadata.Name),
+			zap.String("namespace", canaryConfig.Metadata.Namespace),
+			zap.String("version", canaryConfig.Metadata.ResourceVersion))
+		return
+	}
+	ticker := time.NewTicker(interval)
+
+	// create a context cancel func for each canary config. this will be used to cancel the processing of this canary
+	// config in the event that it's deleted
+	ctx, cancel := context.WithCancel(context.Background())
+
+	cacheValue := &CanaryProcessingInfo{
+		CancelFunc: &cancel,
+		Ticker:     ticker,
+	}
+	err = canaryCfgMgr.canaryCfgCancelFuncMap.assign(&canaryConfig.Metadata, cacheValue)
+	if err != nil {
+		canaryCfgMgr.logger.Error("error caching canary config",
+			zap.Error(err),
+			zap.String("name", canaryConfig.Metadata.Name),
+			zap.String("namespace", canaryConfig.Metadata.Namespace),
+			zap.String("version", canaryConfig.Metadata.ResourceVersion))
+		return
+	}
+	canaryCfgMgr.processCanaryConfig(&ctx, canaryConfig, ticker)
+}
+
+func (canaryCfgMgr *canaryConfigMgr) processCanaryConfig(ctx *context.Context, canaryConfig *fv1.CanaryConfig, ticker *time.Ticker) {
+	quit := make(chan struct{})
+
+	for {
+		select {
+		case <-(*ctx).Done():
+			// this case when someone deleted their canary config in the middle of it being processed
+			canaryCfgMgr.logger.Info("cancel func called for canary config",
+				zap.String("name", canaryConfig.Metadata.Name),
+				zap.String("namespace", canaryConfig.Metadata.Namespace),
+				zap.String("version", canaryConfig.Metadata.ResourceVersion))
+			err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata)
+			if err != nil {
+				canaryCfgMgr.logger.Error("error removing canary config",
+					zap.Error(err),
+					zap.String("name", canaryConfig.Metadata.Name),
+					zap.String("namespace", canaryConfig.Metadata.Namespace),
+					zap.String("version", canaryConfig.Metadata.ResourceVersion))
+			}
+			return
+
+		case <-ticker.C:
+			// every weightIncrementDuration, check if failureThreshold has reached.
+			// if yes, rollback.
+			// else, increment the weight of new function and decrement old function by `weightIncrement`
+			canaryCfgMgr.logger.Info("processing canary config",
+				zap.String("name", canaryConfig.Metadata.Name),
+				zap.String("namespace", canaryConfig.Metadata.Namespace),
+				zap.String("version", canaryConfig.Metadata.ResourceVersion))
+			canaryCfgMgr.RollForwardOrBack(canaryConfig, quit, ticker)
+
+		case <-quit:
+			// we're done processing this canary config either because the new function receives 100% of the traffic
+			// or we rolled back to send all 100% traffic to old function
+			canaryCfgMgr.logger.Info("quit processing canaryConfig",
+				zap.String("name", canaryConfig.Metadata.Name),
+				zap.String("namespace", canaryConfig.Metadata.Namespace),
+				zap.String("version", canaryConfig.Metadata.ResourceVersion))
+			err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata)
+			if err != nil {
+				canaryCfgMgr.logger.Error("error removing canary config from map",
+					zap.Error(err),
+					zap.String("name", canaryConfig.Metadata.Name),
+					zap.String("namespace", canaryConfig.Metadata.Namespace),
+					zap.String("version", canaryConfig.Metadata.ResourceVersion))
+			}
+			return
+		}
+	}
+}
+
+func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *fv1.CanaryConfig, quit chan struct{}, ticker *time.Ticker) {
+	// handle race between delete event and notification on ticker.C
+	_, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
+	if err != nil {
+		canaryCfgMgr.logger.Info("no need of processing the config, not in cache anymore",
+			zap.String("name", canaryConfig.Metadata.Name),
+			zap.String("namespace", canaryConfig.Metadata.Namespace),
+			zap.String("version", canaryConfig.Metadata.ResourceVersion))
+		return
+	}
+
+	// get the http trigger object associated with this canary config
+	triggerObj, err := canaryCfgMgr.fissionClient.HTTPTriggers(canaryConfig.Metadata.Namespace).Get(canaryConfig.Spec.Trigger)
+	if err != nil {
+		// if the http trigger is not found, then give up processing this config.
+		if k8serrors.IsNotFound(err) {
+			canaryCfgMgr.logger.Error("http trigger object for canary config missing",
+				zap.Error(err),
+				zap.String("trigger", canaryConfig.Spec.Trigger),
+				zap.String("name", canaryConfig.Metadata.Name),
+				zap.String("namespace", canaryConfig.Metadata.Namespace),
+				zap.String("version", canaryConfig.Metadata.ResourceVersion))
+			close(quit)
+			return
+		}
+
+		// just silently ignore. wait for next window to increment weight
+		canaryCfgMgr.logger.Error("error fetching http trigger object for config",
+			zap.Error(err),
+			zap.String("name", canaryConfig.Metadata.Name),
+			zap.String("namespace", canaryConfig.Metadata.Namespace),
+			zap.String("version", canaryConfig.Metadata.ResourceVersion))
+		return
+	}
+
+	// handle a race between ticker.Stop and receiving a notification on ticker.C
+	if canaryConfig.Status.Status != types.CanaryConfigStatusPending {
+		canaryCfgMgr.logger.Info("no need of processing the config, not pending anymore",
+			zap.String("name", canaryConfig.Metadata.Name),
+			zap.String("namespace", canaryConfig.Metadata.Namespace),
+			zap.String("version", canaryConfig.Metadata.ResourceVersion))
+		return
+	}
+
+	if triggerObj.Spec.FunctionReference.Type == types.FunctionReferenceTypeFunctionWeights &&
+		triggerObj.Spec.FunctionReference.FunctionWeights[canaryConfig.Spec.NewFunction] != 0 {
+		failurePercent, err := canaryCfgMgr.promClient.GetFunctionFailurePercentage(triggerObj.Spec.RelativeURL, triggerObj.Spec.Method,
+			canaryConfig.Spec.NewFunction, canaryConfig.Metadata.Namespace, canaryConfig.Spec.WeightIncrementDuration)
+
+		if err != nil {
+			// silently ignore. wait for next window to increment weight
+			canaryCfgMgr.logger.Info("error calculating failure percentage",
+				zap.Error(err),
+				zap.String("name", canaryConfig.Metadata.Name),
+				zap.String("namespace", canaryConfig.Metadata.Namespace),
+				zap.String("version", canaryConfig.Metadata.ResourceVersion))
+			return
+		}
+
+		canaryCfgMgr.logger.Info("failure percentage calculated for canaryConfig",
+			zap.Float64("failure_percent", failurePercent),
+			zap.String("name", canaryConfig.Metadata.Name),
+			zap.String("namespace", canaryConfig.Metadata.Namespace),
+			zap.String("version", canaryConfig.Metadata.ResourceVersion))
+
+		if failurePercent == -1 {
+			// this means there were no requests triggered to this url during this window. return here and check back
+			// during next iteration
+			canaryCfgMgr.logger.Info("total requests received for url is 0", zap.String("url", triggerObj.Spec.RelativeURL))
+			return
+		}
+
+		if int(failurePercent) > canaryConfig.Spec.FailureThreshold {
+			canaryCfgMgr.logger.Error("failure percent crossed the threshold, so rolling back",
+				zap.Float64("failure_percent", failurePercent),
+				zap.Int("threshold", canaryConfig.Spec.FailureThreshold),
+				zap.String("name", canaryConfig.Metadata.Name),
+				zap.String("namespace", canaryConfig.Metadata.Namespace),
+				zap.String("version", canaryConfig.Metadata.ResourceVersion))
+			ticker.Stop()
+			err := canaryCfgMgr.rollback(canaryConfig, triggerObj)
+			if err != nil {
+				canaryCfgMgr.logger.Error("error rolling back canary config",
+					zap.String("name", canaryConfig.Metadata.Name),
+					zap.String("namespace", canaryConfig.Metadata.Namespace),
+					zap.String("version", canaryConfig.Metadata.ResourceVersion))
+			}
+			close(quit)
+			return
+		}
+	}
+
+	doneProcessingCanaryConfig, err := canaryCfgMgr.rollForward(canaryConfig, triggerObj)
+	if err != nil {
+		// just log the error and hope that next iteration will succeed
+		canaryCfgMgr.logger.Error("error incrementing weights for trigger",
+			zap.Error(err),
+			zap.String("trigger", triggerObj.Metadata.Name),
+			zap.String("name", canaryConfig.Metadata.Name),
+			zap.String("namespace", canaryConfig.Metadata.Namespace),
+			zap.String("version", canaryConfig.Metadata.ResourceVersion))
+		return
+	}
+
+	if doneProcessingCanaryConfig {
+		ticker.Stop()
+		// update the status of canary config as done processing, we dont care if we arent able to update because
+		// resync takes care of the update
+		err = canaryCfgMgr.updateCanaryConfigStatusWithRetries(canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace,
+			types.CanaryConfigStatusSucceeded)
+		if err != nil {
+			// cant do much after max retries other than logging it.
+			canaryCfgMgr.logger.Error("error updating canary config after max retries",
+				zap.Error(err),
+				zap.String("name", canaryConfig.Metadata.Name),
+				zap.String("namespace", canaryConfig.Metadata.Namespace),
+				zap.String("version", canaryConfig.Metadata.ResourceVersion))
+		}
+
+		canaryCfgMgr.logger.Info("done processing canary config - the new function is receiving all the traffic",
+			zap.String("name", canaryConfig.Metadata.Name),
+			zap.String("namespace", canaryConfig.Metadata.Namespace),
+			zap.String("version", canaryConfig.Metadata.ResourceVersion))
+		close(quit)
+		return
+	}
+}
+
+func (canaryCfgMgr *canaryConfigMgr) updateHttpTriggerWithRetries(triggerName, triggerNamespace string, fnWeights map[string]int) (err error) {
+	for i := 0; i < maxRetries; i++ {
+		triggerObj, err := canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Get(triggerName)
+		if err != nil {
+			e := "error getting http trigger object"
+			canaryCfgMgr.logger.Error(e, zap.Error(err), zap.String("trigger_name", triggerName), zap.String("trigger_namespace", triggerNamespace))
+			return errors.Wrap(err, e)
+		}
+
+		triggerObj.Spec.FunctionReference.FunctionWeights = fnWeights
+
+		_, err = canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Update(triggerObj)
+		switch {
+		case err == nil:
+			canaryCfgMgr.logger.Info("updated http trigger", zap.String("trigger_name", triggerName), zap.String("trigger_namespace", triggerNamespace))
+			return nil
+		case k8serrors.IsConflict(err):
+			canaryCfgMgr.logger.Info("conflict in updating http trigger, retrying",
+				zap.Error(err),
+				zap.String("trigger_name", triggerName),
+				zap.String("trigger_namespace", triggerNamespace))
+			continue
+		default:
+			e := "error updating http trigger"
+			canaryCfgMgr.logger.Info(e,
+				zap.Error(err),
+				zap.String("trigger_name", triggerName),
+				zap.String("trigger_namespace", triggerNamespace))
+			return errors.Wrapf(err, "%s: %s.%s", e, triggerName, triggerNamespace)
+		}
+	}
+
+	return err
+}
+
+func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfigStatusWithRetries(cfgName, cfgNamespace string, status string) (err error) {
+	for i := 0; i < maxRetries; i++ {
+		canaryCfgObj, err := canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Get(cfgName)
+		if err != nil {
+			e := "error getting http canary config object"
+			canaryCfgMgr.logger.Error(e,
+				zap.Error(err),
+				zap.String("name", cfgName),
+				zap.String("namespace", cfgNamespace),
+				zap.String("status", status))
+			return errors.Wrap(err, e)
+		}
+
+		canaryCfgMgr.logger.Info("updating status of canary config",
+			zap.String("name", cfgName),
+			zap.String("namespace", cfgNamespace),
+			zap.String("status", status))
+
+		canaryCfgObj.Status.Status = status
+
+		_, err = canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Update(canaryCfgObj)
+		switch {
+		case err == nil:
+			canaryCfgMgr.logger.Info("updated canary config",
+				zap.String("name", cfgName),
+				zap.String("namespace", cfgNamespace))
+			return nil
+		case k8serrors.IsConflict(err):
+			canaryCfgMgr.logger.Info("conflict in updating canary config",
+				zap.Error(err),
+				zap.String("name", cfgName),
+				zap.String("namespace", cfgNamespace))
+			continue
+		default:
+			e := "error updating canary config"
+			canaryCfgMgr.logger.Error(e,
+				zap.Error(err),
+				zap.String("name", cfgName),
+				zap.String("namespace", cfgNamespace))
+			return errors.Wrapf(err, "%s: %s.%s", e, cfgName, cfgNamespace)
+		}
+	}
+
+	return err
+}
+
+func (canaryCfgMgr *canaryConfigMgr) rollback(canaryConfig *fv1.CanaryConfig, trigger *fv1.HTTPTrigger) error {
+	functionWeights := trigger.Spec.FunctionReference.FunctionWeights
+	functionWeights[canaryConfig.Spec.NewFunction] = 0
+	functionWeights[canaryConfig.Spec.OldFunction] = 100
+
+	err := canaryCfgMgr.updateHttpTriggerWithRetries(trigger.Metadata.Name, trigger.Metadata.Namespace, functionWeights)
+
+	err = canaryCfgMgr.updateCanaryConfigStatusWithRetries(canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace,
+		types.CanaryConfigStatusFailed)
+
+	return err
+}
+
+func (canaryCfgMgr *canaryConfigMgr) rollForward(canaryConfig *fv1.CanaryConfig, trigger *fv1.HTTPTrigger) (bool, error) {
+	doneProcessingCanaryConfig := false
+
+	functionWeights := trigger.Spec.FunctionReference.FunctionWeights
+	if functionWeights[canaryConfig.Spec.NewFunction]+canaryConfig.Spec.WeightIncrement >= 100 {
+		doneProcessingCanaryConfig = true
+		functionWeights[canaryConfig.Spec.NewFunction] = 100
+		functionWeights[canaryConfig.Spec.OldFunction] = 0
+	} else {
+		functionWeights[canaryConfig.Spec.NewFunction] += canaryConfig.Spec.WeightIncrement
+		if functionWeights[canaryConfig.Spec.OldFunction]-canaryConfig.Spec.WeightIncrement < 0 {
+			functionWeights[canaryConfig.Spec.OldFunction] = 0
+		} else {
+			functionWeights[canaryConfig.Spec.OldFunction] -= canaryConfig.Spec.WeightIncrement
+		}
+	}
+
+	canaryCfgMgr.logger.Info("incremented functionWeights", zap.Any("function_weights", functionWeights))
+
+	err := canaryCfgMgr.updateHttpTriggerWithRetries(trigger.Metadata.Name, trigger.Metadata.Namespace, functionWeights)
+	return doneProcessingCanaryConfig, err
+}
+
+func (canaryCfgMgr *canaryConfigMgr) reSyncCanaryConfigs() {
+	for _, obj := range canaryCfgMgr.canaryConfigStore.List() {
+		canaryConfig := obj.(*fv1.CanaryConfig)
+		_, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
+		if err != nil && canaryConfig.Status.Status == types.CanaryConfigStatusPending {
+			canaryCfgMgr.logger.Info("adding canary config from resync loop",
+				zap.String("name", canaryConfig.Metadata.Name),
+				zap.String("namespace", canaryConfig.Metadata.Namespace),
+				zap.String("version", canaryConfig.Metadata.ResourceVersion))
+
+			// new canaryConfig detected, add it to our cache and start processing it
+			go canaryCfgMgr.addCanaryConfig(canaryConfig)
+		}
+	}
+}
+
+func (canaryCfgMgr *canaryConfigMgr) deleteCanaryConfig(canaryConfig *fv1.CanaryConfig) {
+	canaryCfgMgr.logger.Info("delete event received for canary config",
+		zap.String("name", canaryConfig.Metadata.Name),
+		zap.String("namespace", canaryConfig.Metadata.Namespace),
+		zap.String("version", canaryConfig.Metadata.ResourceVersion))
+	canaryProcessingInfo, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
+	if err != nil {
+		canaryCfgMgr.logger.Error("lookup of canary config for deletion failed",
+			zap.Error(err),
+			zap.String("name", canaryConfig.Metadata.Name),
+			zap.String("namespace", canaryConfig.Metadata.Namespace),
+			zap.String("version", canaryConfig.Metadata.ResourceVersion))
+		return
+	}
+	// first stop the ticker
+	canaryProcessingInfo.Ticker.Stop()
+	// call cancel func so that the ctx.Done returns inside processCanaryConfig function and processing gets stopped
+	(*canaryProcessingInfo.CancelFunc)()
+}
+
+func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfig(oldCanaryConfig *fv1.CanaryConfig, newCanaryConfig *fv1.CanaryConfig) {
+	// before removing the object from cache, we need to get it's cancel func and cancel it
+	canaryCfgMgr.deleteCanaryConfig(oldCanaryConfig)
+
+	err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&oldCanaryConfig.Metadata)
+	if err != nil {
+		canaryCfgMgr.logger.Error("error removing canary config from map",
+			zap.Error(err),
+			zap.String("name", oldCanaryConfig.Metadata.Name),
+			zap.String("namespace", oldCanaryConfig.Metadata.Namespace),
+			zap.String("version", oldCanaryConfig.Metadata.ResourceVersion))
+		return
+	}
+	canaryCfgMgr.addCanaryConfig(newCanaryConfig)
+}
+
+func getEnvValue(envVar string) string {
+	envVarSplit := strings.Split(envVar, "=")
+	return envVarSplit[1]
+}

pkg/canaryconfigmgr/prometheusClient.go

@@ -0,0 +1,158 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package canaryconfigmgr
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	promClient "github.com/prometheus/client_golang/api/prometheus"
+	"github.com/prometheus/common/model"
+	"go.uber.org/zap"
+	"golang.org/x/net/context"
+)
+
+type PrometheusApiClient struct {
+	logger *zap.Logger
+	client promClient.QueryAPI
+}
+
+func MakePrometheusClient(logger *zap.Logger, prometheusSvc string) (*PrometheusApiClient, error) {
+	promApiConfig := promClient.Config{
+		Address: prometheusSvc,
+	}
+
+	promApiClient, err := promClient.New(promApiConfig)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error creating prometheus api client for svc: %s", prometheusSvc)
+	}
+
+	apiQueryClient := promClient.NewQueryAPI(promApiClient)
+
+	return &PrometheusApiClient{
+		logger: logger.Named("prometheus_api_client"),
+		client: apiQueryClient,
+	}, nil
+}
+
+func (promApiClient *PrometheusApiClient) GetFunctionFailurePercentage(path, method, funcName, funcNs string, window string) (float64, error) {
+	// first get a total count of requests to this url in a time window
+	reqs, err := promApiClient.GetRequestsToFuncInWindow(path, method, funcName, funcNs, window)
+	if err != nil {
+		return 0, err
+	}
+
+	if reqs <= 0 {
+		return -1, fmt.Errorf("no requests to this url %v and method %v in the window: %v", path, method, window)
+	}
+
+	// next, get a total count of errored out requests to this function in the same window
+	failedReqs, err := promApiClient.GetTotalFailedRequestsToFuncInWindow(funcName, funcNs, path, method, window)
+	if err != nil {
+		return 0, err
+	}
+
+	// calculate the failure percentage of the function
+	failurePercentForFunc := (failedReqs / reqs) * 100
+
+	return failurePercentForFunc, nil
+}
+
+func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string, method string, funcName string, funcNs string, window string) (float64, error) {
+	queryString := fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"}[%v]", path, method, funcName, funcNs, window)
+
+	reqs, err := promApiClient.executeQuery(queryString)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error executing query: %s", queryString)
+	}
+
+	queryString = fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"} offset %v", path, method, funcName, funcNs, window)
+
+	reqsInPrevWindow, err := promApiClient.executeQuery(queryString)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error executing query: %s", queryString)
+	}
+
+	reqsInCurrentWindow := reqs - reqsInPrevWindow
+	promApiClient.logger.Info("function requests",
+		zap.Float64("requests", reqs),
+		zap.Float64("requests_in_previous_window", reqsInPrevWindow),
+		zap.Float64("requests_in_current_window", reqsInCurrentWindow),
+		zap.String("function", funcName))
+
+	return reqsInCurrentWindow, nil
+}
+
+func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(funcName string, funcNs string, path string, method string, window string) (float64, error) {
+	queryString := fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"}[%v]", funcName, funcNs, path, method, window)
+
+	failedRequests, err := promApiClient.executeQuery(queryString)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error executing query: %s", queryString)
+	}
+
+	queryString = fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"} offset %v", funcName, funcNs, path, method, window)
+
+	failedReqsInPrevWindow, err := promApiClient.executeQuery(queryString)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error executing query: %s", queryString)
+	}
+
+	failedReqsInCurrentWindow := failedRequests - failedReqsInPrevWindow
+	promApiClient.logger.Info("function requests",
+		zap.Float64("failed_requests", failedRequests),
+		zap.Float64("failed_requests_in_previous_window", failedReqsInPrevWindow),
+		zap.Float64("failed_requests_in_current_window", failedReqsInCurrentWindow),
+		zap.String("function", funcName))
+
+	return failedReqsInCurrentWindow, nil
+}
+
+func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (float64, error) {
+	val, err := promApiClient.client.Query(context.Background(), queryString, time.Now())
+	if err != nil {
+		return 0, errors.Wrapf(err, "error querying prometheus")
+	}
+
+	switch {
+	case val.Type() == model.ValScalar:
+		scalarVal := val.(*model.Scalar)
+		return float64(scalarVal.Value), nil
+
+	case val.Type() == model.ValVector:
+		vectorVal := val.(model.Vector)
+		total := float64(0)
+		for _, elem := range vectorVal {
+			total = total + float64(elem.Value)
+		}
+		return total, nil
+
+	case val.Type() == model.ValMatrix:
+		matrixVal := val.(model.Matrix)
+		total := float64(0)
+		for _, elem := range matrixVal {
+			total += float64(elem.Values[len(elem.Values)-1].Value)
+		}
+		return total, nil
+
+	default:
+		promApiClient.logger.Info("return value type of prometheus query was unrecognized",
+			zap.Any("type", val.Type()))
+		return 0, nil
+	}
+}

pkg/controller/api.go

@@ -0,0 +1,280 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"runtime/debug"
+	"strings"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/fission/fission/pkg/crd"
+	ferror "github.com/fission/fission/pkg/error"
+	"github.com/fission/fission/pkg/fission-cli/logdb"
+	"github.com/fission/fission/pkg/info"
+	"github.com/fission/fission/pkg/utils"
+)
+
+var podNamespace string
+
+func init() {
+	podNamespace = os.Getenv("POD_NAMESPACE")
+	if podNamespace == "" {
+		podNamespace = "fission"
+	}
+}
+
+type (
+	API struct {
+		logger            *zap.Logger
+		fissionClient     *crd.FissionClient
+		kubernetesClient  *kubernetes.Clientset
+		storageServiceUrl string
+		builderManagerUrl string
+		workflowApiUrl    string
+		functionNamespace string
+		useIstio          bool
+		featureStatus     map[string]string
+	}
+
+	logDBConfig struct {
+		httpURL  string
+		username string
+		password string
+	}
+)
+
+func MakeAPI(logger *zap.Logger, featureStatus map[string]string) (*API, error) {
+	api, err := makeCRDBackedAPI(logger)
+
+	u := os.Getenv("STORAGE_SERVICE_URL")
+	if len(u) > 0 {
+		api.storageServiceUrl = strings.TrimSuffix(u, "/")
+	} else {
+		api.storageServiceUrl = "http://storagesvc"
+	}
+
+	u = os.Getenv("BUILDER_MANAGER_URL")
+	if len(u) > 0 {
+		api.builderManagerUrl = strings.TrimSuffix(u, "/")
+	} else {
+		api.builderManagerUrl = "http://buildermgr"
+	}
+
+	wfEnv := os.Getenv("WORKFLOW_API_URL")
+	if len(u) > 0 {
+		api.workflowApiUrl = strings.TrimSuffix(wfEnv, "/")
+	} else {
+		api.workflowApiUrl = "http://workflows-apiserver"
+	}
+
+	fnNs := os.Getenv("FISSION_FUNCTION_NAMESPACE")
+	if len(fnNs) > 0 {
+		api.functionNamespace = fnNs
+	} else {
+		api.functionNamespace = "fission-function"
+	}
+
+	api.featureStatus = featureStatus
+
+	return api, err
+}
+
+func (api *API) respondWithSuccess(w http.ResponseWriter, resp []byte) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	_, err := w.Write(resp)
+	if err != nil {
+		// this will probably fail too, but try anyway
+		api.respondWithError(w, err)
+	}
+}
+
+func (api *API) respondWithError(w http.ResponseWriter, err error) {
+	debug.PrintStack()
+
+	// this error type comes with an HTTP code, so just use that
+	se, ok := err.(*kerrors.StatusError)
+	if ok {
+		http.Error(w, string(se.ErrStatus.Reason), int(se.ErrStatus.Code))
+		return
+	}
+
+	code, msg := ferror.GetHTTPError(err)
+	api.logger.Error(msg, zap.Int("code", code))
+	http.Error(w, msg, code)
+}
+
+func (api *API) extractQueryParamFromRequest(r *http.Request, queryParam string) string {
+	values := r.URL.Query()
+	return values.Get(queryParam)
+}
+
+// check if namespace exists, if not create it.
+func (api *API) createNsIfNotExists(ns string) error {
+	if ns == metav1.NamespaceDefault {
+		// we dont have to create default ns
+		return nil
+	}
+
+	_, err := api.kubernetesClient.CoreV1().Namespaces().Get(ns, metav1.GetOptions{})
+	if err != nil && kerrors.IsNotFound(err) {
+		ns := &apiv1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: ns,
+			},
+		}
+		_, err = api.kubernetesClient.CoreV1().Namespaces().Create(ns)
+	}
+
+	return err
+}
+
+func (api *API) getLogDBConfig(dbType string) logDBConfig {
+	dbType = strings.ToUpper(dbType)
+	// retrieve db auth config from the env
+	url := os.Getenv(fmt.Sprintf("%s_URL", dbType))
+	if url == "" {
+		// set up default database url
+		url = logdb.INFLUXDB_URL
+	}
+	username := os.Getenv(fmt.Sprintf("%s_USERNAME", dbType))
+	password := os.Getenv(fmt.Sprintf("%s_PASSWORD", dbType))
+	return logDBConfig{
+		httpURL:  url,
+		username: username,
+		password: password,
+	}
+}
+
+func (api *API) HomeHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	fmt.Fprintf(w, info.ApiInfo().String())
+}
+
+func (api *API) ApiVersionMismatchHandler(w http.ResponseWriter, r *http.Request) {
+	err := ferror.MakeError(ferror.ErrorNotFound, "Fission server supports API v2 only -- v1 is not supported. Please upgrade your Fission client/CLI.")
+	api.respondWithError(w, err)
+}
+
+func (api *API) HealthHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+func (api *API) GetSvcName(w http.ResponseWriter, r *http.Request) {
+	appLabelSelector := "application=" + r.URL.Query().Get("application")
+	services, err := api.kubernetesClient.CoreV1().Services(podNamespace).List(metav1.ListOptions{
+		LabelSelector: appLabelSelector,
+	})
+	if err != nil || len(services.Items) > 1 || len(services.Items) == 0 {
+		api.respondWithError(w, err)
+	}
+	service := services.Items[0]
+	fmt.Fprintf(w, service.Name+"."+podNamespace)
+}
+
+func (api *API) Serve(port int) {
+	r := mux.NewRouter()
+	r.HandleFunc("/healthz", api.HealthHandler).Methods("GET")
+	// Give a useful error message if an older CLI attempts to make a request
+	r.HandleFunc(`/v1/{rest:[a-zA-Z0-9=\-\/]+}`, api.ApiVersionMismatchHandler)
+	r.HandleFunc("/", api.HomeHandler)
+
+	r.HandleFunc("/v2/packages", api.PackageApiList).Methods("GET")
+	r.HandleFunc("/v2/packages", api.PackageApiCreate).Methods("POST")
+	r.HandleFunc("/v2/packages/{package}", api.PackageApiGet).Methods("GET")
+	r.HandleFunc("/v2/packages/{package}", api.PackageApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/packages/{package}", api.PackageApiDelete).Methods("DELETE")
+
+	r.HandleFunc("/v2/functions", api.FunctionApiList).Methods("GET")
+	r.HandleFunc("/v2/functions", api.FunctionApiCreate).Methods("POST")
+	r.HandleFunc("/v2/functions/{function}", api.FunctionApiGet).Methods("GET")
+	r.HandleFunc("/v2/functions/{function}", api.FunctionApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/functions/{function}", api.FunctionApiDelete).Methods("DELETE")
+
+	r.HandleFunc("/v2/triggers/http", api.HTTPTriggerApiList).Methods("GET")
+	r.HandleFunc("/v2/triggers/http", api.HTTPTriggerApiCreate).Methods("POST")
+	r.HandleFunc("/v2/triggers/http/{httpTrigger}", api.HTTPTriggerApiGet).Methods("GET")
+	r.HandleFunc("/v2/triggers/http/{httpTrigger}", api.HTTPTriggerApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/triggers/http/{httpTrigger}", api.HTTPTriggerApiDelete).Methods("DELETE")
+
+	r.HandleFunc("/v2/environments", api.EnvironmentApiList).Methods("GET")
+	r.HandleFunc("/v2/environments", api.EnvironmentApiCreate).Methods("POST")
+	r.HandleFunc("/v2/environments/{environment}", api.EnvironmentApiGet).Methods("GET")
+	r.HandleFunc("/v2/environments/{environment}", api.EnvironmentApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/environments/{environment}", api.EnvironmentApiDelete).Methods("DELETE")
+
+	r.HandleFunc("/v2/watches", api.WatchApiList).Methods("GET")
+	r.HandleFunc("/v2/watches", api.WatchApiCreate).Methods("POST")
+	r.HandleFunc("/v2/watches/{watch}", api.WatchApiGet).Methods("GET")
+	r.HandleFunc("/v2/watches/{watch}", api.WatchApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/watches/{watch}", api.WatchApiDelete).Methods("DELETE")
+
+	r.HandleFunc("/v2/triggers/time", api.TimeTriggerApiList).Methods("GET")
+	r.HandleFunc("/v2/triggers/time", api.TimeTriggerApiCreate).Methods("POST")
+	r.HandleFunc("/v2/triggers/time/{timeTrigger}", api.TimeTriggerApiGet).Methods("GET")
+	r.HandleFunc("/v2/triggers/time/{timeTrigger}", api.TimeTriggerApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/triggers/time/{timeTrigger}", api.TimeTriggerApiDelete).Methods("DELETE")
+
+	r.HandleFunc("/v2/triggers/messagequeue", api.MessageQueueTriggerApiList).Methods("GET")
+	r.HandleFunc("/v2/triggers/messagequeue", api.MessageQueueTriggerApiCreate).Methods("POST")
+	r.HandleFunc("/v2/triggers/messagequeue/{mqTrigger}", api.MessageQueueTriggerApiGet).Methods("GET")
+	r.HandleFunc("/v2/triggers/messagequeue/{mqTrigger}", api.MessageQueueTriggerApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/triggers/messagequeue/{mqTrigger}", api.MessageQueueTriggerApiDelete).Methods("DELETE")
+
+	r.HandleFunc("/v2/recorders", api.RecorderApiList).Methods("GET")
+	r.HandleFunc("/v2/recorders", api.RecorderApiCreate).Methods("POST")
+	r.HandleFunc("/v2/recorders/{recorder}", api.RecorderApiGet).Methods("GET")
+	r.HandleFunc("/v2/recorders/{recorder}", api.RecorderApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/recorders/{recorder}", api.RecorderApiDelete).Methods("DELETE")
+
+	r.HandleFunc("/v2/records", api.RecordsApiListAll).Methods("GET")
+	r.HandleFunc("/v2/records/function/{function}", api.RecordsApiFilterByFunction).Methods("GET")
+	r.HandleFunc("/v2/records/trigger/{trigger}", api.RecordsApiFilterByTrigger).Methods("GET")
+	r.HandleFunc("/v2/records/time", api.RecordsApiFilterByTime).Methods("GET")
+
+	r.HandleFunc("/v2/replay/{reqUID}", api.ReplayByReqUID).Methods("GET")
+
+	r.HandleFunc("/v2/secrets/{secret}", api.SecretGet).Methods("GET")
+	r.HandleFunc("/v2/configmaps/{configmap}", api.ConfigMapGet).Methods("GET")
+
+	r.HandleFunc("/v2/canaryconfigs", api.CanaryConfigApiCreate).Methods("POST")
+	r.HandleFunc("/v2/canaryconfigs/{canaryConfig}", api.CanaryConfigApiGet).Methods("GET")
+	r.HandleFunc("/v2/canaryconfigs/{canaryConfig}", api.CanaryConfigApiUpdate).Methods("PUT")
+	r.HandleFunc("/v2/canaryconfigs/{canaryConfig}", api.CanaryConfigApiDelete).Methods("DELETE")
+	r.HandleFunc("/v2/canaryconfigs", api.CanaryConfigApiList).Methods("GET")
+
+	r.HandleFunc("/proxy/{dbType}", api.FunctionLogsApiPost).Methods("POST")
+	r.HandleFunc("/proxy/storage/v1/archive", api.StorageServiceProxy)
+	r.HandleFunc("/proxy/logs/{function}", api.FunctionPodLogs).Methods("POST")
+	r.HandleFunc("/proxy/workflows-apiserver/{path:.*}", api.WorkflowApiserverProxy)
+	r.HandleFunc("/proxy/svcname", api.GetSvcName).Queries("application", "").Methods("GET")
+
+	address := fmt.Sprintf(":%v", port)
+
+	api.logger.Info("server started", zap.Int("port", port))
+	r.Use(utils.LoggingMiddleware(api.logger))
+	err := http.ListenAndServe(address, r)
+	api.logger.Fatal("done listening", zap.Error(err))
+}

pkg/controller/api_test.go

@@ -0,0 +1,372 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/controller/client"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+var g struct {
+	client *client.Client
+}
+
+func panicIf(err error) {
+	if err != nil {
+		log.Panicf("err: %v", err)
+	}
+}
+
+func assert(c bool, msg string) {
+	if !c {
+		log.Fatalf("assert failed: %v", msg)
+	}
+}
+
+func assertNameReuseFailure(err error, name string) {
+	assert(err != nil, "recreating "+name+" with same name must fail")
+	fe, ok := err.(ferror.Error)
+	assert(ok, "error must be a fission Error")
+	assert(fe.Code == ferror.ErrorNameExists, "error must be a name exists error")
+}
+
+func assertNotFoundFailure(err error, name string) {
+	assert(err != nil, "requesting a non-existent "+name+" must fail")
+	fe, ok := err.(ferror.Error)
+	assert(ok, "error must be a fission Error")
+	if fe.Code != ferror.ErrorNotFound {
+		log.Fatalf("error must be a not found error: %v", fe)
+	}
+}
+
+func assertCronSpecFails(err error) {
+	assert(err != nil, "using an invalid cron spec must fail")
+	ok := strings.Contains(err.Error(), "not a valid cron spec")
+	assert(ok, "invalid cron spec must fail")
+}
+
+func TestFunctionApi(t *testing.T) {
+	testFunc := &fv1.Function{
+		Metadata: metav1.ObjectMeta{
+			Name:      "foo",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: fv1.FunctionSpec{
+			Environment: fv1.EnvironmentReference{
+				Name:      "nodejs",
+				Namespace: metav1.NamespaceDefault,
+			},
+			Package: fv1.FunctionPackageRef{
+				FunctionName: "xxx",
+				PackageRef: fv1.PackageRef{
+					Namespace:       metav1.NamespaceDefault,
+					Name:            "xxx",
+					ResourceVersion: "12345",
+				},
+			},
+		},
+	}
+	_, err := g.client.FunctionGet(&metav1.ObjectMeta{
+		Name:      testFunc.Metadata.Name,
+		Namespace: metav1.NamespaceDefault,
+	})
+	assertNotFoundFailure(err, "function")
+
+	m, err := g.client.FunctionCreate(testFunc)
+	panicIf(err)
+	defer func() {
+		err := g.client.FunctionDelete(m)
+		panicIf(err)
+	}()
+
+	_, err = g.client.FunctionCreate(testFunc)
+	assertNameReuseFailure(err, "function")
+
+	testFunc.Metadata.ResourceVersion = m.ResourceVersion
+	testFunc.Spec.Package.FunctionName = "yyy"
+	_, err = g.client.FunctionUpdate(testFunc)
+	panicIf(err)
+
+	testFunc.Metadata.ResourceVersion = ""
+	testFunc.Metadata.Name = "bar"
+	m2, err := g.client.FunctionCreate(testFunc)
+	panicIf(err)
+	defer g.client.FunctionDelete(m2)
+
+	funcs, err := g.client.FunctionList(metav1.NamespaceDefault)
+	panicIf(err)
+	assert(len(funcs) == 2, fmt.Sprintf("created two functions, but found %v", len(funcs)))
+
+	funcs_url := g.client.Url + "/v2/functions"
+	resp, err := http.Get(funcs_url)
+	panicIf(err)
+	defer resp.Body.Close()
+	assert(resp.StatusCode == 200, "http get status code on /v1/functions")
+
+	var found bool = false
+	for _, b := range resp.Header["Content-Type"] {
+		if b == "application/json; charset=utf-8" {
+			found = true
+		}
+	}
+	assert(found, "incorrect response content type")
+}
+
+func TestHTTPTriggerApi(t *testing.T) {
+	testTrigger := &fv1.HTTPTrigger{
+		Metadata: metav1.ObjectMeta{
+			Name:      "foo",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: fv1.HTTPTriggerSpec{
+			Method:      http.MethodGet,
+			RelativeURL: "/hello",
+			FunctionReference: fv1.FunctionReference{
+				Type: fv1.FunctionReferenceTypeFunctionName,
+				Name: "foo",
+			},
+		},
+	}
+	_, err := g.client.HTTPTriggerGet(&metav1.ObjectMeta{
+		Name:      testTrigger.Metadata.Name,
+		Namespace: metav1.NamespaceDefault,
+	})
+	assertNotFoundFailure(err, "httptrigger")
+
+	m, err := g.client.HTTPTriggerCreate(testTrigger)
+	panicIf(err)
+	defer g.client.HTTPTriggerDelete(m)
+
+	_, err = g.client.HTTPTriggerCreate(testTrigger)
+	assertNameReuseFailure(err, "httptrigger")
+
+	tr, err := g.client.HTTPTriggerGet(m)
+	panicIf(err)
+	assert(testTrigger.Spec.Method == tr.Spec.Method &&
+		testTrigger.Spec.RelativeURL == tr.Spec.RelativeURL &&
+		testTrigger.Spec.FunctionReference.Type == tr.Spec.FunctionReference.Type &&
+		testTrigger.Spec.FunctionReference.Name == tr.Spec.FunctionReference.Name, "trigger should match after reading")
+
+	testTrigger.Metadata.ResourceVersion = m.ResourceVersion
+	testTrigger.Spec.RelativeURL = "/hi"
+	_, err = g.client.HTTPTriggerUpdate(testTrigger)
+	panicIf(err)
+
+	testTrigger.Metadata.ResourceVersion = ""
+	testTrigger.Metadata.Name = "yyy"
+	_, err = g.client.HTTPTriggerCreate(testTrigger)
+	assert(err != nil, "duplicate trigger should not be allowed")
+
+	testTrigger.Spec.RelativeURL = "/hi2"
+	m2, err := g.client.HTTPTriggerCreate(testTrigger)
+	panicIf(err)
+	defer g.client.HTTPTriggerDelete(m2)
+
+	ts, err := g.client.HTTPTriggerList(metav1.NamespaceDefault)
+	panicIf(err)
+	assert(len(ts) == 2, fmt.Sprintf("created two triggers, but found %v", len(ts)))
+}
+
+func TestEnvironmentApi(t *testing.T) {
+
+	testEnv := &fv1.Environment{
+		Metadata: metav1.ObjectMeta{
+			Name:      "foo",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: fv1.EnvironmentSpec{
+			Runtime: fv1.Runtime{
+				Image: "gcr.io/xyz",
+			},
+			Resources: v1.ResourceRequirements{},
+		},
+	}
+	_, err := g.client.EnvironmentGet(&metav1.ObjectMeta{
+		Name:      testEnv.Metadata.Name,
+		Namespace: metav1.NamespaceDefault,
+	})
+	assertNotFoundFailure(err, "environment")
+
+	m, err := g.client.EnvironmentCreate(testEnv)
+	panicIf(err)
+	defer g.client.EnvironmentDelete(m)
+
+	_, err = g.client.EnvironmentCreate(testEnv)
+	assertNameReuseFailure(err, "environment")
+
+	e, err := g.client.EnvironmentGet(m)
+	panicIf(err)
+	assert(reflect.DeepEqual(testEnv.Spec, e.Spec), "env should match after reading")
+
+	testEnv.Metadata.ResourceVersion = m.ResourceVersion
+	testEnv.Spec.Runtime.Image = "another-img"
+	_, err = g.client.EnvironmentUpdate(testEnv)
+	panicIf(err)
+
+	testEnv.Metadata.ResourceVersion = ""
+	testEnv.Metadata.Name = "bar"
+	m2, err := g.client.EnvironmentCreate(testEnv)
+	panicIf(err)
+	defer g.client.EnvironmentDelete(m2)
+
+	ts, err := g.client.EnvironmentList(metav1.NamespaceDefault)
+	panicIf(err)
+	assert(len(ts) == 2, fmt.Sprintf("created two envs, but found %v", len(ts)))
+}
+
+func TestWatchApi(t *testing.T) {
+	testWatch := &fv1.KubernetesWatchTrigger{
+		Metadata: metav1.ObjectMeta{
+			Name:      "xxx",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: fv1.KubernetesWatchTriggerSpec{
+			Namespace: "default",
+			Type:      "pod",
+			FunctionReference: fv1.FunctionReference{
+				Type: fv1.FunctionReferenceTypeFunctionName,
+				Name: "foo",
+			},
+		},
+	}
+	_, err := g.client.WatchGet(&metav1.ObjectMeta{
+		Name:      testWatch.Metadata.Name,
+		Namespace: metav1.NamespaceDefault,
+	})
+	assertNotFoundFailure(err, "watch")
+
+	m, err := g.client.WatchCreate(testWatch)
+	panicIf(err)
+	defer g.client.WatchDelete(m)
+
+	_, err = g.client.WatchCreate(testWatch)
+	assertNameReuseFailure(err, "watch")
+
+	w, err := g.client.WatchGet(m)
+	panicIf(err)
+	assert(testWatch.Spec.Namespace == w.Spec.Namespace &&
+		testWatch.Spec.Type == w.Spec.Type &&
+		testWatch.Spec.FunctionReference.Type == w.Spec.FunctionReference.Type &&
+		testWatch.Spec.FunctionReference.Name == w.Spec.FunctionReference.Name, "watch should match after reading")
+
+	testWatch.Metadata.Name = "yyy"
+	m2, err := g.client.WatchCreate(testWatch)
+	panicIf(err)
+	defer g.client.WatchDelete(m2)
+
+	ws, err := g.client.WatchList(metav1.NamespaceDefault)
+	panicIf(err)
+	assert(len(ws) == 2, fmt.Sprintf("created two watches, but found %v", len(ws)))
+}
+
+func TestTimeTriggerApi(t *testing.T) {
+	testTrigger := &fv1.TimeTrigger{
+		Metadata: metav1.ObjectMeta{
+			Name:      "xxx",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: fv1.TimeTriggerSpec{
+			Cron: "0 30 * * * *",
+			FunctionReference: fv1.FunctionReference{
+				Type: fv1.FunctionReferenceTypeFunctionName,
+				Name: "asdf",
+			},
+		},
+	}
+	_, err := g.client.TimeTriggerGet(&metav1.ObjectMeta{Name: testTrigger.Metadata.Name})
+	assertNotFoundFailure(err, "trigger")
+
+	m, err := g.client.TimeTriggerCreate(testTrigger)
+	panicIf(err)
+	defer g.client.TimeTriggerDelete(m)
+
+	_, err = g.client.TimeTriggerCreate(testTrigger)
+	assertNameReuseFailure(err, "trigger")
+
+	tr, err := g.client.TimeTriggerGet(m)
+	panicIf(err)
+	assert(testTrigger.Spec.Cron == tr.Spec.Cron &&
+		testTrigger.Spec.FunctionReference.Type == tr.Spec.FunctionReference.Type &&
+		testTrigger.Spec.FunctionReference.Name == tr.Spec.FunctionReference.Name, "trigger should match after reading")
+
+	testTrigger.Metadata.ResourceVersion = m.ResourceVersion
+	testTrigger.Spec.Cron = "@hourly"
+	_, err = g.client.TimeTriggerUpdate(testTrigger)
+	panicIf(err)
+
+	testTrigger.Metadata.ResourceVersion = ""
+	testTrigger.Metadata.Name = "yyy"
+	testTrigger.Spec.Cron = "Not valid cron spec"
+	_, err = g.client.TimeTriggerCreate(testTrigger)
+	assertCronSpecFails(err)
+
+	ts, err := g.client.TimeTriggerList(metav1.NamespaceDefault)
+	panicIf(err)
+	assert(len(ts) == 1, fmt.Sprintf("created two time triggers, but found %v", len(ts)))
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// skip test if no cluster available for testing
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if len(kubeconfig) == 0 {
+		log.Println("Skipping test, no kubernetes cluster")
+		return
+	}
+
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	go Start(logger, 8888, true)
+
+	time.Sleep(5 * time.Second)
+	g.client = client.MakeClient("http://localhost:8888")
+
+	resp, err := http.Get("http://localhost:8888/")
+	panicIf(err)
+	assert(resp.StatusCode == 200, "http get status code on root")
+
+	var found bool = false
+	for _, b := range resp.Header["Content-Type"] {
+		if b == "application/json; charset=utf-8" {
+			found = true
+		}
+	}
+	assert(found, "incorrect response content type")
+
+	_, err = ioutil.ReadAll(resp.Body)
+	panicIf(err)
+
+	os.Exit(m.Run())
+}

pkg/controller/canaryConfigApi.go

@@ -0,0 +1,184 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+	config "github.com/fission/fission/pkg/featureconfig"
+)
+
+func (a *API) CanaryConfigApiCreate(w http.ResponseWriter, r *http.Request) {
+	featureErr := a.featureStatus[config.CanaryFeature]
+	if len(featureErr) > 0 {
+		a.respondWithError(w, ferror.MakeError(http.StatusInternalServerError, fmt.Sprintf("Error enabling canary feature: %v", featureErr)))
+		return
+	}
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var canaryCfg fv1.CanaryConfig
+	err = json.Unmarshal(body, &canaryCfg)
+	if err != nil {
+		a.logger.Error("failed to unmarshal request body", zap.Error(err), zap.Binary("body", body))
+		a.respondWithError(w, err)
+		return
+	}
+
+	canaryCfgNew, err := a.fissionClient.CanaryConfigs(canaryCfg.Metadata.Namespace).Create(&canaryCfg)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(canaryCfgNew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) CanaryConfigApiGet(w http.ResponseWriter, r *http.Request) {
+	featureErr := a.featureStatus[config.CanaryFeature]
+	if len(featureErr) > 0 {
+		a.respondWithError(w, ferror.MakeError(http.StatusInternalServerError, fmt.Sprintf("Error enabling canary feature: %v", featureErr)))
+		return
+	}
+
+	vars := mux.Vars(r)
+	name := vars["canaryConfig"]
+
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	canaryCfg, err := a.fissionClient.CanaryConfigs(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(canaryCfg)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) CanaryConfigApiList(w http.ResponseWriter, r *http.Request) {
+	featureErr := a.featureStatus[config.CanaryFeature]
+	if len(featureErr) > 0 {
+		a.respondWithError(w, ferror.MakeError(http.StatusInternalServerError, fmt.Sprintf("Error enabling canary feature: %v", featureErr)))
+		return
+	}
+
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	canaryCfgs, err := a.fissionClient.CanaryConfigs(ns).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(canaryCfgs.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) CanaryConfigApiUpdate(w http.ResponseWriter, r *http.Request) {
+	featureErr := a.featureStatus[config.CanaryFeature]
+	if len(featureErr) > 0 {
+		a.respondWithError(w, ferror.MakeError(http.StatusInternalServerError, fmt.Sprintf("Error enabling canary feature: %v", featureErr)))
+		return
+	}
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var c fv1.CanaryConfig
+	err = json.Unmarshal(body, &c)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	canayCfgNew, err := a.fissionClient.CanaryConfigs(c.Metadata.Namespace).Update(&c)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(canayCfgNew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) CanaryConfigApiDelete(w http.ResponseWriter, r *http.Request) {
+	featureErr := a.featureStatus[config.CanaryFeature]
+	if len(featureErr) > 0 {
+		a.respondWithError(w, ferror.MakeError(http.StatusInternalServerError, fmt.Sprintf("Error enabling canary feature: %v", featureErr)))
+		return
+	}
+
+	vars := mux.Vars(r)
+	name := vars["canaryConfig"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.CanaryConfigs(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, []byte(""))
+}

pkg/controller/client/canaryconfig.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func (c *Client) CanaryConfigCreate(canaryConf *fv1.CanaryConfig) (*metav1.ObjectMeta, error) {
+	reqbody, err := json.Marshal(canaryConf)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("canaryconfigs"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) CanaryConfigGet(m *metav1.ObjectMeta) (*fv1.CanaryConfig, error) {
+	relativeUrl := fmt.Sprintf("canaryconfigs/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var canaryCfg fv1.CanaryConfig
+	err = json.Unmarshal(body, &canaryCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &canaryCfg, nil
+}
+
+func (c *Client) CanaryConfigUpdate(canaryConf *fv1.CanaryConfig) (*metav1.ObjectMeta, error) {
+	reqbody, err := json.Marshal(canaryConf)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("canaryconfigs/%v", canaryConf.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) CanaryConfigDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("canaryconfigs/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) CanaryConfigList(ns string) ([]fv1.CanaryConfig, error) {
+	relativeUrl := fmt.Sprintf("canaryconfigs?namespace=%v", ns)
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	canaryCfgs := make([]fv1.CanaryConfig, 0)
+	err = json.Unmarshal(body, &canaryCfgs)
+	if err != nil {
+		return nil, err
+	}
+
+	return canaryCfgs, nil
+}

pkg/controller/client/client.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	ferror "github.com/fission/fission/pkg/error"
+	"github.com/fission/fission/pkg/info"
+)
+
+type (
+	Client struct {
+		Url string
+	}
+)
+
+func MakeClient(serverUrl string) *Client {
+	return &Client{Url: strings.TrimSuffix(serverUrl, "/")}
+}
+
+func (c *Client) delete(relativeUrl string) error {
+	req, err := http.NewRequest("DELETE", c.url(relativeUrl), nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return errors.New("Delete failed")
+		} else {
+			return errors.New("Delete failed: " + string(body))
+		}
+	}
+
+	return nil
+}
+
+func (c *Client) put(relativeUrl string, contentType string, body []byte) (*http.Response, error) {
+	req, err := http.NewRequest("PUT", c.url(relativeUrl), bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-type", contentType)
+	return http.DefaultClient.Do(req)
+}
+
+func (c *Client) url(relativeUrl string) string {
+	return c.Url + "/v2/" + relativeUrl
+}
+
+func (c *Client) handleResponse(resp *http.Response) ([]byte, error) {
+	if resp.StatusCode != 200 {
+		return nil, ferror.MakeErrorFromHTTP(resp)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	return body, err
+}
+
+func (c *Client) handleCreateResponse(resp *http.Response) ([]byte, error) {
+	if resp.StatusCode != 201 {
+		return nil, ferror.MakeErrorFromHTTP(resp)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	return body, err
+}
+
+func (c *Client) ServerInfo() (*info.ServerInfo, error) {
+	url := fmt.Sprintf(c.Url)
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	info := &info.ServerInfo{}
+	err = json.Unmarshal(body, info)
+	if err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}

pkg/controller/client/core.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (c *Client) SecretGet(m *metav1.ObjectMeta) (*apiv1.Secret, error) {
+	relativeUrl := fmt.Sprintf("secrets/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var secret apiv1.Secret
+	err = json.Unmarshal(body, &secret)
+	if err != nil {
+		return nil, err
+	}
+
+	return &secret, nil
+}
+
+func (c *Client) ConfigMapGet(m *metav1.ObjectMeta) (*apiv1.ConfigMap, error) {
+	relativeUrl := fmt.Sprintf("configmaps/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var configMap apiv1.ConfigMap
+	err = json.Unmarshal(body, &configMap)
+	if err != nil {
+		return nil, err
+	}
+
+	return &configMap, nil
+}
+
+func (c *Client) GetSvcURL(label string) (string, error) {
+	url := fmt.Sprintf("%s/proxy/svcname?"+label, c.Url)
+
+	resp, err := http.Get(url)
+
+	if err != nil {
+		return "", err
+	}
+
+	if resp == nil {
+		return "", fmt.Errorf("Failed to find service for given label: %v", label)
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	storageSvc := string(body)
+
+	return storageSvc, err
+}

pkg/controller/client/environment.go

@@ -0,0 +1,143 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func (c *Client) EnvironmentCreate(env *fv1.Environment) (*metav1.ObjectMeta, error) {
+	err := env.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("Environment", err)
+	}
+
+	reqbody, err := json.Marshal(env)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("environments"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) EnvironmentGet(m *metav1.ObjectMeta) (*fv1.Environment, error) {
+	relativeUrl := fmt.Sprintf("environments/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var env fv1.Environment
+	err = json.Unmarshal(body, &env)
+	if err != nil {
+		return nil, err
+	}
+
+	return &env, nil
+}
+
+func (c *Client) EnvironmentUpdate(env *fv1.Environment) (*metav1.ObjectMeta, error) {
+	err := env.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("Environment", err)
+	}
+
+	reqbody, err := json.Marshal(env)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("environments/%v", env.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) EnvironmentDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("environments/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) EnvironmentList(ns string) ([]fv1.Environment, error) {
+	relativeUrl := fmt.Sprintf("environments?namespace=%v", ns)
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	envs := make([]fv1.Environment, 0)
+	err = json.Unmarshal(body, &envs)
+	if err != nil {
+		return nil, err
+	}
+
+	return envs, nil
+}

pkg/controller/client/function.go

@@ -0,0 +1,156 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func (c *Client) FunctionCreate(f *fv1.Function) (*metav1.ObjectMeta, error) {
+	err := f.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("Function", err)
+	}
+
+	reqbody, err := json.Marshal(f)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("functions"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) FunctionGet(m *metav1.ObjectMeta) (*fv1.Function, error) {
+	relativeUrl := fmt.Sprintf("functions/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var f fv1.Function
+	err = json.Unmarshal(body, &f)
+	if err != nil {
+		return nil, err
+	}
+
+	return &f, nil
+}
+
+func (c *Client) FunctionGetRawDeployment(m *metav1.ObjectMeta) ([]byte, error) {
+	relativeUrl := fmt.Sprintf("functions/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+	relativeUrl += fmt.Sprintf("&deploymentraw=1")
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	return c.handleResponse(resp)
+}
+
+func (c *Client) FunctionUpdate(f *fv1.Function) (*metav1.ObjectMeta, error) {
+	err := f.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("Function", err)
+	}
+
+	reqbody, err := json.Marshal(f)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("functions/%v", f.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) FunctionDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("functions/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) FunctionList(functionNamespace string) ([]fv1.Function, error) {
+	relativeUrl := fmt.Sprintf("functions?namespace=%v", functionNamespace)
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	funcs := make([]fv1.Function, 0)
+	err = json.Unmarshal(body, &funcs)
+	if err != nil {
+		return nil, err
+	}
+
+	return funcs, nil
+}

pkg/controller/client/httptrigger.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func (c *Client) HTTPTriggerCreate(t *fv1.HTTPTrigger) (*metav1.ObjectMeta, error) {
+	err := t.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("HTTPTrigger", err)
+	}
+
+	reqbody, err := json.Marshal(t)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("triggers/http"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) HTTPTriggerGet(m *metav1.ObjectMeta) (*fv1.HTTPTrigger, error) {
+	relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var t fv1.HTTPTrigger
+	err = json.Unmarshal(body, &t)
+	if err != nil {
+		return nil, err
+	}
+
+	return &t, nil
+}
+
+func (c *Client) HTTPTriggerUpdate(t *fv1.HTTPTrigger) (*metav1.ObjectMeta, error) {
+	err := t.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("HTTPTrigger", err)
+	}
+
+	reqbody, err := json.Marshal(t)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("triggers/http/%v", t.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) HTTPTriggerDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("triggers/http/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) HTTPTriggerList(triggerNamespace string) ([]fv1.HTTPTrigger, error) {
+	relativeUrl := fmt.Sprintf("triggers/http?namespace=%v", triggerNamespace)
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	triggers := make([]fv1.HTTPTrigger, 0)
+	err = json.Unmarshal(body, &triggers)
+	if err != nil {
+		return nil, err
+	}
+
+	return triggers, nil
+}

pkg/controller/client/kuberneteswatchtrigger.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (c *Client) WatchCreate(w *fv1.KubernetesWatchTrigger) (*metav1.ObjectMeta, error) {
+	err := w.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("KubernetesWatchTrigger", err)
+	}
+
+	reqbody, err := json.Marshal(w)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("watches"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) WatchGet(m *metav1.ObjectMeta) (*fv1.KubernetesWatchTrigger, error) {
+	relativeUrl := fmt.Sprintf("watches/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var w fv1.KubernetesWatchTrigger
+	err = json.Unmarshal(body, &w)
+	if err != nil {
+		return nil, err
+	}
+
+	return &w, nil
+}
+
+func (c *Client) WatchUpdate(w *fv1.KubernetesWatchTrigger) (*metav1.ObjectMeta, error) {
+	return nil, ferror.MakeError(ferror.ErrorNotImplmented,
+		"watch update not implemented")
+}
+
+func (c *Client) WatchDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("watches/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) WatchList(ns string) ([]fv1.KubernetesWatchTrigger, error) {
+	relativeUrl := fmt.Sprintf("watches?namespace=%v", ns)
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	watches := make([]fv1.KubernetesWatchTrigger, 0)
+	err = json.Unmarshal(body, &watches)
+	if err != nil {
+		return nil, err
+	}
+
+	return watches, err
+}

pkg/controller/client/mqtrigger.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func (c *Client) MessageQueueTriggerCreate(t *fv1.MessageQueueTrigger) (*metav1.ObjectMeta, error) {
+	err := t.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("MessageQueueTrigger", err)
+	}
+
+	reqbody, err := json.Marshal(t)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("triggers/messagequeue"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) MessageQueueTriggerGet(m *metav1.ObjectMeta) (*fv1.MessageQueueTrigger, error) {
+	relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var t fv1.MessageQueueTrigger
+	err = json.Unmarshal(body, &t)
+	if err != nil {
+		return nil, err
+	}
+
+	return &t, nil
+}
+
+func (c *Client) MessageQueueTriggerUpdate(mqTrigger *fv1.MessageQueueTrigger) (*metav1.ObjectMeta, error) {
+	err := mqTrigger.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("MessageQueueTrigger", err)
+	}
+
+	reqbody, err := json.Marshal(mqTrigger)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", mqTrigger.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) MessageQueueTriggerDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("triggers/messagequeue/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) MessageQueueTriggerList(mqType string, ns string) ([]fv1.MessageQueueTrigger, error) {
+	relativeUrl := "triggers/messagequeue"
+	if len(mqType) > 0 {
+		// TODO remove this, replace with field selector
+		relativeUrl += fmt.Sprintf("?mqtype=%v&namespace=%v", mqType, ns)
+	}
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	triggers := make([]fv1.MessageQueueTrigger, 0)
+	err = json.Unmarshal(body, &triggers)
+	if err != nil {
+		return nil, err
+	}
+
+	return triggers, nil
+}

pkg/controller/client/package.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func (c *Client) PackageCreate(f *fv1.Package) (*metav1.ObjectMeta, error) {
+	err := f.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("Package", err)
+	}
+
+	reqbody, err := json.Marshal(f)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("packages"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) PackageGet(m *metav1.ObjectMeta) (*fv1.Package, error) {
+	relativeUrl := fmt.Sprintf("packages/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var f fv1.Package
+	err = json.Unmarshal(body, &f)
+	if err != nil {
+		return nil, err
+	}
+
+	return &f, nil
+}
+
+func (c *Client) PackageUpdate(f *fv1.Package) (*metav1.ObjectMeta, error) {
+	err := f.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("Package", err)
+	}
+
+	reqbody, err := json.Marshal(f)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("packages/%v", f.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) PackageDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("packages/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) PackageList(pkgNamespace string) ([]fv1.Package, error) {
+	relativeUrl := fmt.Sprintf("packages?namespace=%v", pkgNamespace)
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	funcs := make([]fv1.Package, 0)
+	err = json.Unmarshal(body, &funcs)
+	if err != nil {
+		return nil, err
+	}
+
+	return funcs, nil
+}

pkg/controller/client/recorder.go

@@ -0,0 +1,238 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/redis/build/gen"
+)
+
+func (c *Client) RecorderCreate(r *fv1.Recorder) (*metav1.ObjectMeta, error) {
+	err := r.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("Recorder", err)
+	}
+
+	reqbody, err := json.Marshal(r)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("recorders"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) RecorderGet(m *metav1.ObjectMeta) (*fv1.Recorder, error) {
+	relativeUrl := fmt.Sprintf("recorders/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var r fv1.Recorder
+	err = json.Unmarshal(body, &r)
+	if err != nil {
+		return nil, err
+	}
+
+	return &r, nil
+}
+
+func (c *Client) RecorderUpdate(recorder *fv1.Recorder) (*metav1.ObjectMeta, error) {
+	err := recorder.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("Recorder", err)
+	}
+
+	reqbody, err := json.Marshal(recorder)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("recorders/%v", recorder.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) RecorderDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("recorders/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) RecorderList(ns string) ([]fv1.Recorder, error) {
+	relativeUrl := "recorders"
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	recorders := make([]fv1.Recorder, 0)
+	err = json.Unmarshal(body, &recorders)
+	if err != nil {
+		return nil, err
+	}
+
+	return recorders, nil
+}
+
+// TODO: Move to different file?
+func (c *Client) RecordsByFunction(function string) ([]*redisCache.RecordedEntry, error) {
+	relativeUrl := fmt.Sprintf("records/function/%v", function)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	records := make([]*redisCache.RecordedEntry, 0)
+	err = json.Unmarshal(body, &records)
+	if err != nil {
+		return nil, err
+	}
+
+	return records, nil
+}
+
+func (c *Client) RecordsAll() ([]*redisCache.RecordedEntry, error) {
+	relativeUrl := "records"
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	records := make([]*redisCache.RecordedEntry, 0)
+	err = json.Unmarshal(body, &records)
+	if err != nil {
+		return nil, err
+	}
+
+	return records, nil
+}
+
+func (c *Client) RecordsByTrigger(trigger string) ([]*redisCache.RecordedEntry, error) {
+	relativeUrl := fmt.Sprintf("records/trigger/%v", trigger)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	records := make([]*redisCache.RecordedEntry, 0)
+	err = json.Unmarshal(body, &records)
+	if err != nil {
+		return nil, err
+	}
+
+	return records, nil
+}
+
+func (c *Client) RecordsByTime(from string, to string) ([]*redisCache.RecordedEntry, error) {
+	relativeUrl := "records/time"
+	relativeUrl += fmt.Sprintf("?from=%v&to=%v", from, to)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	records := make([]*redisCache.RecordedEntry, 0)
+	err = json.Unmarshal(body, &records)
+	if err != nil {
+		return nil, err
+	}
+
+	return records, nil
+}

pkg/controller/client/replayer.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+func (c *Client) ReplayByReqUID(reqUID string) ([]string, error) {
+	relativeUrl := fmt.Sprintf("replay/%v", reqUID)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	replayed := make([]string, 0)
+
+	err = json.Unmarshal(body, &replayed)
+	if err != nil {
+		return nil, err
+	}
+
+	return replayed, nil
+}

pkg/controller/client/timetrigger.go

@@ -0,0 +1,142 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func (c *Client) TimeTriggerCreate(t *fv1.TimeTrigger) (*metav1.ObjectMeta, error) {
+	err := t.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("TimeTrigger", err)
+	}
+
+	reqbody, err := json.Marshal(t)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := http.Post(c.url("triggers/time"), "application/json", bytes.NewReader(reqbody))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleCreateResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (c *Client) TimeTriggerGet(m *metav1.ObjectMeta) (*fv1.TimeTrigger, error) {
+	relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var t fv1.TimeTrigger
+	err = json.Unmarshal(body, &t)
+	if err != nil {
+		return nil, err
+	}
+
+	return &t, nil
+}
+
+func (c *Client) TimeTriggerUpdate(t *fv1.TimeTrigger) (*metav1.ObjectMeta, error) {
+	err := t.Validate()
+	if err != nil {
+		return nil, fv1.AggregateValidationErrors("TimeTrigger", err)
+	}
+
+	reqbody, err := json.Marshal(t)
+	if err != nil {
+		return nil, err
+	}
+	relativeUrl := fmt.Sprintf("triggers/time/%v", t.Metadata.Name)
+
+	resp, err := c.put(relativeUrl, "application/json", reqbody)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	var m metav1.ObjectMeta
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		return nil, err
+	}
+	return &m, nil
+}
+
+func (c *Client) TimeTriggerDelete(m *metav1.ObjectMeta) error {
+	relativeUrl := fmt.Sprintf("triggers/time/%v", m.Name)
+	relativeUrl += fmt.Sprintf("?namespace=%v", m.Namespace)
+	return c.delete(relativeUrl)
+}
+
+func (c *Client) TimeTriggerList(ns string) ([]fv1.TimeTrigger, error) {
+	relativeUrl := fmt.Sprintf("triggers/time?namespace=%v", ns)
+	resp, err := http.Get(c.url(relativeUrl))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := c.handleResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	triggers := make([]fv1.TimeTrigger, 0)
+	err = json.Unmarshal(body, &triggers)
+	if err != nil {
+		return nil, err
+	}
+
+	return triggers, nil
+}

pkg/controller/config.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/fission/fission/pkg/canaryconfigmgr"
+	"github.com/fission/fission/pkg/crd"
+	config "github.com/fission/fission/pkg/featureconfig"
+)
+
+func ConfigCanaryFeature(context context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, featureConfig *config.FeatureConfig, featureStatus map[string]string) error {
+	// start the appropriate controller
+	if featureConfig.CanaryConfig.IsEnabled {
+		canaryCfgMgr, err := canaryconfigmgr.MakeCanaryConfigMgr(logger, fissionClient, kubeClient, fissionClient.GetCrdClient(),
+			featureConfig.CanaryConfig.PrometheusSvc)
+		if err != nil {
+			featureStatus[config.CanaryFeature] = err.Error()
+			return errors.Wrap(err, "failed to start canary config manager")
+		}
+		canaryCfgMgr.Run(context)
+		logger.Info("started canary config manager")
+	}
+
+	return nil
+}
+
+// ConfigureFeatures gets the feature config and configures the features that are enabled
+func ConfigureFeatures(context context.Context, logger *zap.Logger, unitTestMode bool, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset) (map[string]string, error) {
+	// set feature enabled to false if unitTestMode
+	if unitTestMode {
+		return nil, nil
+	}
+
+	// get the featureConfig from config map mounted onto the file system
+	featureConfig, err := config.GetFeatureConfig()
+	if err != nil {
+		logger.Error("error getting feature config", zap.Error(err))
+		return nil, err
+	}
+
+	featureStatus := make(map[string]string)
+
+	// configure respective features
+	// in the future when new optional features are added, we need to add corresponding feature handlers and invoke them here
+	err = ConfigCanaryFeature(context, logger, fissionClient, kubeClient, featureConfig, featureStatus)
+	return featureStatus, err
+}

pkg/controller/configmapApi.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (a *API) ConfigMapGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["configmap"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	configMap, err := a.kubernetesClient.CoreV1().ConfigMaps(ns).Get(name, metav1.GetOptions{})
+	if err != nil {
+		a.logger.Error("error getting config map", zap.Error(err), zap.String("config_map_name", name), zap.String("namespace", ns))
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(configMap)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}

pkg/controller/controller.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/utils"
+)
+
+func Start(logger *zap.Logger, port int, unitTestFlag bool) {
+	cLogger := logger.Named("controller")
+	// setup a signal handler for SIGTERM
+	utils.SetupStackTraceHandler()
+
+	fc, kc, apiExtClient, err := crd.MakeFissionClient()
+	if err != nil {
+		cLogger.Fatal("failed to connect to k8s API", zap.Error(err))
+	}
+
+	err = crd.EnsureFissionCRDs(cLogger, apiExtClient)
+	if err != nil {
+		cLogger.Fatal("failed to create fission CRDs", zap.Error(err))
+	}
+
+	err = fc.WaitForCRDs()
+	if err != nil {
+		cLogger.Fatal("error waiting for CRDs", zap.Error(err))
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc)
+	if err != nil {
+		cLogger.Info("error configuring features - proceeding without optional features", zap.Error(err))
+	}
+	defer cancel()
+
+	api, err := MakeAPI(cLogger, featureStatus)
+	if err != nil {
+		cLogger.Fatal("failed to start controller", zap.Error(err))
+	}
+	api.Serve(port)
+}

pkg/controller/crd.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/crd"
+)
+
+func makeCRDBackedAPI(logger *zap.Logger) (*API, error) {
+	fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
+	if err != nil {
+		return nil, err
+	}
+	return &API{
+		logger:           logger.Named("api"),
+		fissionClient:    fissionClient,
+		kubernetesClient: kubernetesClient,
+	}, nil
+}

pkg/controller/environmentApi.go

@@ -0,0 +1,169 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (a *API) EnvironmentApiList(w http.ResponseWriter, r *http.Request) {
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceAll
+	}
+
+	envs, err := a.fissionClient.Environments(ns).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(envs.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) EnvironmentApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var env fv1.Environment
+	err = json.Unmarshal(body, &env)
+	if err != nil {
+		a.logger.Error("failed to unmarshal request body", zap.Error(err), zap.Binary("body", body))
+		a.respondWithError(w, err)
+		return
+	}
+
+	// check if namespace exists, if not create it.
+	err = a.createNsIfNotExists(env.Metadata.Namespace)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	enew, err := a.fissionClient.Environments(env.Metadata.Namespace).Create(&env)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(enew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) EnvironmentApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["environment"]
+
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	env, err := a.fissionClient.Environments(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(env)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) EnvironmentApiUpdate(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["environment"]
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var env fv1.Environment
+	err = json.Unmarshal(body, &env)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	if name != env.Metadata.Name {
+		err = ferror.MakeError(ferror.ErrorInvalidArgument, "Environment name doesn't match URL")
+		a.respondWithError(w, err)
+		return
+	}
+
+	enew, err := a.fissionClient.Environments(env.Metadata.Namespace).Update(&env)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(enew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) EnvironmentApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["environment"]
+
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.Environments(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, []byte(""))
+}

pkg/controller/functionApi.go

@@ -0,0 +1,271 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"sort"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	restclient "k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (a *API) getIstioServiceLabels(fnName string) map[string]string {
+	return map[string]string{
+		"functionName": fnName,
+	}
+}
+
+func (a *API) FunctionApiList(w http.ResponseWriter, r *http.Request) {
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceAll
+	}
+
+	funcs, err := a.fissionClient.Functions(ns).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(funcs.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) FunctionApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var f fv1.Function
+	err = json.Unmarshal(body, &f)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	// check if namespace exists, if not create it.
+	err = a.createNsIfNotExists(f.Metadata.Namespace)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	fnew, err := a.fissionClient.Functions(f.Metadata.Namespace).Create(&f)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(fnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) FunctionApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["function"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	f, err := a.fissionClient.Functions(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(f)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) FunctionApiUpdate(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["function"]
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var f fv1.Function
+	err = json.Unmarshal(body, &f)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	if name != f.Metadata.Name {
+		err = ferror.MakeError(ferror.ErrorInvalidArgument, "Function name doesn't match URL")
+		a.respondWithError(w, err)
+		return
+	}
+
+	fnew, err := a.fissionClient.Functions(f.Metadata.Namespace).Update(&f)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(fnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) FunctionApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["function"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.Functions(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, []byte(""))
+}
+
+// FunctionLogsApiPost establishes a proxy server to log database, and redirect
+// query command send from client to database then proxy back the db response.
+func (a *API) FunctionLogsApiPost(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	// get dbType from url
+	dbType := vars["dbType"]
+
+	// find correspond db http url
+	dbCnf := a.getLogDBConfig(dbType)
+
+	svcUrl, err := url.Parse(dbCnf.httpURL)
+	if err != nil {
+		a.logger.Error("failed parse url to establish proxy to database for function logs",
+			zap.Error(err),
+			zap.String("database_url", dbCnf.httpURL))
+	}
+	// set up proxy server director
+	director := func(req *http.Request) {
+		// only replace url Scheme and Host to remote influxDB
+		// and leave query string intact
+		req.URL.Scheme = svcUrl.Scheme
+		req.URL.Host = svcUrl.Host
+		req.URL.Path = svcUrl.Path
+		// set up http basic auth for database authentication
+		req.SetBasicAuth(dbCnf.username, dbCnf.password)
+	}
+	proxy := &httputil.ReverseProxy{
+		Director: director,
+	}
+	proxy.ServeHTTP(w, r)
+}
+
+// FunctionPodLogs : Get logs for a function directly from pod
+func (a *API) FunctionPodLogs(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	fnName := vars["function"]
+	ns := vars["namespace"]
+
+	if len(ns) == 0 {
+		ns = "fission-function"
+	}
+
+	f, err := a.fissionClient.Functions(ns).Get(fnName)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	envName := f.Spec.Environment.Name
+
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	// Get function Pods first
+	selector := "functionName=" + fnName
+	podList, err := a.kubernetesClient.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: selector})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	// Get the logs for last Pod executed
+	pods := podList.Items
+	sort.Slice(pods, func(i, j int) bool {
+		itime := pods[i].ObjectMeta.CreationTimestamp.Time
+		jtime := pods[j].ObjectMeta.CreationTimestamp.Time
+		return itime.After(jtime)
+	})
+
+	podLogOpts := apiv1.PodLogOptions{Container: envName} // Only the env container, not fetcher
+	var podLogsReq *restclient.Request
+	if len(pods) > 0 {
+		podLogsReq = a.kubernetesClient.CoreV1().Pods(ns).GetLogs(pods[0].ObjectMeta.Name, &podLogOpts)
+	} else {
+		a.respondWithError(w, errors.New("No active pods found"))
+		return
+	}
+
+	podLogs, err := podLogsReq.Stream()
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	defer podLogs.Close()
+
+	_, err = io.Copy(w, podLogs)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	return
+}

pkg/controller/httpTriggerApi.go

@@ -0,0 +1,198 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (a *API) HTTPTriggerApiList(w http.ResponseWriter, r *http.Request) {
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceAll
+	}
+
+	triggers, err := a.fissionClient.HTTPTriggers(ns).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(triggers.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+// checkHTTPTriggerDuplicates checks whether the tuple (Method, Host, URL) is duplicate or not.
+func (a *API) checkHTTPTriggerDuplicates(t *fv1.HTTPTrigger) error {
+	triggers, err := a.fissionClient.HTTPTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, ht := range triggers.Items {
+		if ht.Metadata.UID == t.Metadata.UID {
+			// Same resource. No need to check.
+			continue
+		}
+		if ht.Spec.RelativeURL == t.Spec.RelativeURL && ht.Spec.Method == t.Spec.Method && ht.Spec.Host == t.Spec.Host {
+			return ferror.MakeError(ferror.ErrorNameExists,
+				fmt.Sprintf("HTTPTrigger with same Host, URL & method already exists (%v)",
+					ht.Metadata.Name))
+		}
+	}
+	return nil
+}
+
+func (a *API) HTTPTriggerApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var t fv1.HTTPTrigger
+	err = json.Unmarshal(body, &t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	// Ensure we don't have a duplicate HTTP route defined (same URL and method)
+	err = a.checkHTTPTriggerDuplicates(&t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	// check if namespace exists, if not create it.
+	err = a.createNsIfNotExists(t.Metadata.Namespace)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	tnew, err := a.fissionClient.HTTPTriggers(t.Metadata.Namespace).Create(&t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(tnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) HTTPTriggerApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["httpTrigger"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	t, err := a.fissionClient.HTTPTriggers(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) HTTPTriggerApiUpdate(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["httpTrigger"]
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var t fv1.HTTPTrigger
+	err = json.Unmarshal(body, &t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	if name != t.Metadata.Name {
+		err = ferror.MakeError(ferror.ErrorInvalidArgument, "HTTPTrigger name doesn't match URL")
+		a.respondWithError(w, err)
+		return
+	}
+
+	err = a.checkHTTPTriggerDuplicates(&t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	tnew, err := a.fissionClient.HTTPTriggers(t.Metadata.Namespace).Update(&t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(tnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) HTTPTriggerApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["httpTrigger"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.HTTPTriggers(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, []byte(""))
+}

pkg/controller/mqTriggerApi.go

@@ -0,0 +1,159 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (a *API) MessageQueueTriggerApiList(w http.ResponseWriter, r *http.Request) {
+	//mqType := r.FormValue("mqtype") // ignored for now
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceAll
+	}
+
+	triggers, err := a.fissionClient.MessageQueueTriggers(ns).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	resp, err := json.Marshal(triggers.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) MessageQueueTriggerApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var mqTrigger fv1.MessageQueueTrigger
+	err = json.Unmarshal(body, &mqTrigger)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	// check if namespace exists, if not create it.
+	err = a.createNsIfNotExists(mqTrigger.Metadata.Namespace)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	tnew, err := a.fissionClient.MessageQueueTriggers(mqTrigger.Metadata.Namespace).Create(&mqTrigger)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(tnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) MessageQueueTriggerApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["mqTrigger"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	mqTrigger, err := a.fissionClient.MessageQueueTriggers(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	resp, err := json.Marshal(mqTrigger)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) MessageQueueTriggerApiUpdate(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["mqTrigger"]
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var mqTrigger fv1.MessageQueueTrigger
+	err = json.Unmarshal(body, &mqTrigger)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	if name != mqTrigger.Metadata.Name {
+		err = ferror.MakeError(ferror.ErrorInvalidArgument, "Message queue trigger name doesn't match URL")
+		a.respondWithError(w, err)
+		return
+	}
+
+	tnew, err := a.fissionClient.MessageQueueTriggers(mqTrigger.Metadata.Namespace).Update(&mqTrigger)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(tnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) MessageQueueTriggerApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["mqTrigger"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.MessageQueueTriggers(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, []byte(""))
+}

pkg/controller/packageApi.go

@@ -0,0 +1,185 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/dustin/go-humanize"
+	"github.com/fission/fission/pkg/types"
+	"github.com/gorilla/mux"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (a *API) PackageApiList(w http.ResponseWriter, r *http.Request) {
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceAll
+	}
+	funcs, err := a.fissionClient.Packages(ns).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(funcs.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) PackageApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var f fv1.Package
+	err = json.Unmarshal(body, &f)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	// Ensure size limits
+	if len(f.Spec.Source.Literal) > int(types.ArchiveLiteralSizeLimit) {
+		err := ferror.MakeError(ferror.ErrorInvalidArgument,
+			fmt.Sprintf("Package literal larger than %s", humanize.Bytes(uint64(types.ArchiveLiteralSizeLimit))))
+		a.respondWithError(w, err)
+		return
+	}
+	if len(f.Spec.Deployment.Literal) > int(types.ArchiveLiteralSizeLimit) {
+		err := ferror.MakeError(ferror.ErrorInvalidArgument,
+			fmt.Sprintf("Package literal larger than %s", humanize.Bytes(uint64(types.ArchiveLiteralSizeLimit))))
+		a.respondWithError(w, err)
+		return
+	}
+
+	// check if namespace exists, if not create it.
+	err = a.createNsIfNotExists(f.Metadata.Namespace)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	fnew, err := a.fissionClient.Packages(f.Metadata.Namespace).Create(&f)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(fnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) PackageApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["package"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+	raw := r.FormValue("raw") // just the deployment pkg
+
+	f, err := a.fissionClient.Packages(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var resp []byte
+	if raw != "" {
+		resp = []byte(f.Spec.Deployment.Literal)
+	} else {
+		resp, err = json.Marshal(f)
+		if err != nil {
+			a.respondWithError(w, err)
+			return
+		}
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) PackageApiUpdate(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["package"]
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var f fv1.Package
+	err = json.Unmarshal(body, &f)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	if name != f.Metadata.Name {
+		err = ferror.MakeError(ferror.ErrorInvalidArgument, "Package name doesn't match URL")
+		a.respondWithError(w, err)
+		return
+	}
+
+	fnew, err := a.fissionClient.Packages(f.Metadata.Namespace).Update(&f)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(fnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) PackageApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["package"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.Packages(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, []byte(""))
+}

pkg/controller/recorderApi.go

@@ -0,0 +1,148 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (a *API) RecorderApiList(w http.ResponseWriter, r *http.Request) {
+	recorders, err := a.fissionClient.Recorders(metav1.NamespaceAll).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	resp, err := json.Marshal(recorders.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) RecorderApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var recorder fv1.Recorder
+	err = json.Unmarshal(body, &recorder)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	tnew, err := a.fissionClient.Recorders(recorder.Metadata.Namespace).Create(&recorder)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(tnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) RecorderApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["recorder"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	recorder, err := a.fissionClient.Recorders(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	resp, err := json.Marshal(recorder)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) RecorderApiUpdate(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["recorder"]
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var recorder fv1.Recorder
+	err = json.Unmarshal(body, &recorder)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	if name != recorder.Metadata.Name {
+		err = ferror.MakeError(ferror.ErrorInvalidArgument, "Recorder name doesn't match URL")
+		a.respondWithError(w, err)
+		return
+	}
+
+	rnew, err := a.fissionClient.Recorders(recorder.Metadata.Namespace).Update(&recorder)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(rnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) RecorderApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["recorder"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.Recorders(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, []byte(""))
+}

pkg/controller/recordsApi.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission/pkg/redis"
+)
+
+func (a *API) RecordsApiListAll(w http.ResponseWriter, r *http.Request) {
+	resp, err := redis.RecordsListAll(a.logger.Named("redis"))
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) RecordsApiFilterByFunction(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	query := vars["function"]
+
+	recorders, err := a.fissionClient.Recorders(metav1.NamespaceAll).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	triggers, err := a.fissionClient.HTTPTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := redis.RecordsFilterByFunction(a.logger.Named("redis"), query, recorders, triggers)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) RecordsApiFilterByTrigger(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	query := vars["trigger"]
+
+	recorders, err := a.fissionClient.Recorders(metav1.NamespaceAll).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	triggers, err := a.fissionClient.HTTPTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := redis.RecordsFilterByTrigger(a.logger.Named("redis"), query, recorders, triggers)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) RecordsApiFilterByTime(w http.ResponseWriter, r *http.Request) {
+	from := r.FormValue("from")
+	to := r.FormValue("to")
+
+	resp, err := redis.RecordsFilterByTime(a.logger.Named("redis"), from, to)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}

pkg/controller/replayAPI.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/gorilla/mux"
+
+	"github.com/fission/fission/pkg/redis"
+)
+
+func (a *API) ReplayByReqUID(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	queriedID := vars["reqUID"]
+
+	routerUrl := fmt.Sprintf("http://router.%v", podNamespace)
+
+	resp, err := redis.ReplayByReqUID(a.logger, routerUrl, queriedID)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}

pkg/controller/secretApi.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func (a *API) SecretGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["secret"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	secret, err := a.kubernetesClient.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{})
+	if err != nil {
+		a.logger.Error("error getting secret",
+			zap.Error(err),
+			zap.String("secret_name", name),
+			zap.String("namespace", ns))
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(secret)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}

pkg/controller/storagesvc.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+
+	"go.uber.org/zap"
+)
+
+func (api *API) StorageServiceProxy(w http.ResponseWriter, r *http.Request) {
+	u := api.storageServiceUrl
+	ssUrl, err := url.Parse(u)
+	if err != nil {
+		e := "error parsing url"
+		api.logger.Error(e, zap.Error(err), zap.String("url", u))
+		http.Error(w, fmt.Sprintf("%s %s: %v", e, u, err), http.StatusInternalServerError)
+		return
+	}
+	director := func(req *http.Request) {
+		req.URL.Scheme = ssUrl.Scheme
+		req.URL.Host = ssUrl.Host
+		req.URL.Path = "/v1/archive"
+	}
+	proxy := &httputil.ReverseProxy{
+		Director: director,
+	}
+	proxy.ServeHTTP(w, r)
+}

pkg/controller/timeTriggerApi.go

@@ -0,0 +1,180 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/robfig/cron"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (a *API) TimeTriggerApiList(w http.ResponseWriter, r *http.Request) {
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceAll
+	}
+
+	triggers, err := a.fissionClient.TimeTriggers(ns).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(triggers.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) TimeTriggerApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var t fv1.TimeTrigger
+	err = json.Unmarshal(body, &t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	// validate
+	_, err = cron.Parse(t.Spec.Cron)
+	if err != nil {
+		err = ferror.MakeError(ferror.ErrorInvalidArgument, "TimeTrigger cron spec is not valid")
+		a.respondWithError(w, err)
+		return
+	}
+
+	// check if namespace exists, if not create it.
+	err = a.createNsIfNotExists(t.Metadata.Namespace)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	tnew, err := a.fissionClient.TimeTriggers(t.Metadata.Namespace).Create(&t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(tnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) TimeTriggerApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["timeTrigger"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	t, err := a.fissionClient.TimeTriggers(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) TimeTriggerApiUpdate(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["timeTrigger"]
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var t fv1.TimeTrigger
+	err = json.Unmarshal(body, &t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	if name != t.Metadata.Name {
+		err = ferror.MakeError(ferror.ErrorInvalidArgument, "TimeTrigger name doesn't match URL")
+		a.respondWithError(w, err)
+		return
+	}
+
+	_, err = cron.Parse(t.Spec.Cron)
+	if err != nil {
+		err = ferror.MakeError(ferror.ErrorInvalidArgument, "TimeTrigger cron spec is not valid")
+		a.respondWithError(w, err)
+		return
+	}
+
+	tnew, err := a.fissionClient.TimeTriggers(t.Metadata.Namespace).Update(&t)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(tnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) TimeTriggerApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["timeTrigger"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.TimeTriggers(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, []byte(""))
+}

pkg/controller/watchApi.go

@@ -0,0 +1,134 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (a *API) WatchApiList(w http.ResponseWriter, r *http.Request) {
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceAll
+	}
+
+	watches, err := a.fissionClient.KubernetesWatchTriggers(ns).List(metav1.ListOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(watches.Items)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) WatchApiCreate(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	var watch fv1.KubernetesWatchTrigger
+	err = json.Unmarshal(body, &watch)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	// TODO check for duplicate watches
+	// TODO check for duplicate watches -> we probably wont need it?
+	// check if namespace exists, if not create it.
+	err = a.createNsIfNotExists(watch.Metadata.Namespace)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	wnew, err := a.fissionClient.KubernetesWatchTriggers(watch.Metadata.Namespace).Create(&watch)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(wnew.Metadata)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	w.WriteHeader(http.StatusCreated)
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) WatchApiGet(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["watch"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	watch, err := a.fissionClient.KubernetesWatchTriggers(ns).Get(name)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	resp, err := json.Marshal(watch)
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, resp)
+}
+
+func (a *API) WatchApiUpdate(w http.ResponseWriter, r *http.Request) {
+	a.respondWithError(w, ferror.MakeError(ferror.ErrorNotImplmented,
+		"Not implemented"))
+}
+
+func (a *API) WatchApiDelete(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	name := vars["watch"]
+	ns := a.extractQueryParamFromRequest(r, "namespace")
+	if len(ns) == 0 {
+		ns = metav1.NamespaceDefault
+	}
+
+	err := a.fissionClient.KubernetesWatchTriggers(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		a.respondWithError(w, err)
+		return
+	}
+
+	a.respondWithSuccess(w, []byte(""))
+}

pkg/controller/workflowApiProxy.go

@@ -0,0 +1,32 @@
+package controller
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+
+	"github.com/gorilla/mux"
+)
+
+func (api *API) WorkflowApiserverProxy(w http.ResponseWriter, r *http.Request) {
+	u := api.workflowApiUrl
+	ssUrl, err := url.Parse(u)
+	if err != nil {
+		msg := fmt.Sprintf("Error parsing url %v: %v", u, err)
+		http.Error(w, msg, http.StatusInternalServerError)
+		return
+	}
+
+	vars := mux.Vars(r)
+	path := fmt.Sprintf("/%s", vars["path"])
+	director := func(req *http.Request) {
+		req.URL.Scheme = ssUrl.Scheme
+		req.URL.Host = ssUrl.Host
+		req.URL.Path = path
+	}
+	proxy := &httputil.ReverseProxy{
+		Director: director,
+	}
+	proxy.ServeHTTP(w, r)
+}

pkg/crd/canaryConfig.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	CanaryConfigInterface interface {
+		Create(*fv1.CanaryConfig) (*fv1.CanaryConfig, error)
+		Get(name string) (*fv1.CanaryConfig, error)
+		Update(*fv1.CanaryConfig) (*fv1.CanaryConfig, error)
+		Delete(name string, options *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*fv1.CanaryConfigList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	canaryConfigClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakeCanaryConfigInterface(crdClient *rest.RESTClient, namespace string) CanaryConfigInterface {
+	return &canaryConfigClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (c *canaryConfigClient) Create(f *fv1.CanaryConfig) (*fv1.CanaryConfig, error) {
+	var result fv1.CanaryConfig
+	err := c.client.Post().
+		Resource("canaryconfigs").
+		Namespace(c.namespace).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *canaryConfigClient) Get(name string) (*fv1.CanaryConfig, error) {
+	var result fv1.CanaryConfig
+	err := c.client.Get().
+		Resource("canaryconfigs").
+		Namespace(c.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *canaryConfigClient) Update(f *fv1.CanaryConfig) (*fv1.CanaryConfig, error) {
+	var result fv1.CanaryConfig
+	err := c.client.Put().
+		Resource("canaryconfigs").
+		Namespace(c.namespace).
+		Name(f.Metadata.Name).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *canaryConfigClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return c.client.Delete().
+		Namespace(c.namespace).
+		Resource("canaryconfigs").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (c *canaryConfigClient) List(opts metav1.ListOptions) (*fv1.CanaryConfigList, error) {
+	var result fv1.CanaryConfigList
+	err := c.client.Get().
+		Namespace(c.namespace).
+		Resource("canaryconfigs").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *canaryConfigClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return c.client.Get().
+		Prefix("watch").
+		Namespace(c.namespace).
+		Resource("canaryconfigs").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

pkg/crd/client.go

@@ -0,0 +1,241 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	"errors"
+	"os"
+	"time"
+
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	FissionClient struct {
+		crdClient *rest.RESTClient
+	}
+)
+
+// Get a kubernetes client using the kubeconfig file at the
+// environment var $KUBECONFIG, or an in-cluster config if that's
+// undefined.
+func GetKubernetesClient() (*rest.Config, *kubernetes.Clientset, *apiextensionsclient.Clientset, error) {
+	var config *rest.Config
+	var err error
+
+	// get the config, either from kubeconfig or using our
+	// in-cluster service account
+	kubeConfig := os.Getenv("KUBECONFIG")
+	if len(kubeConfig) != 0 {
+		config, err = clientcmd.BuildConfigFromFlags("", kubeConfig)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+	} else {
+		config, err = rest.InClusterConfig()
+		if err != nil {
+			return nil, nil, nil, err
+		}
+	}
+
+	// creates the clientset
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	apiExtClientset, err := apiextensionsclient.NewForConfig(config)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return config, clientset, apiExtClientset, nil
+}
+
+// GetCrdClient gets a CRD client config
+func GetCrdClient(config *rest.Config) (*rest.RESTClient, error) {
+	// mutate config to add our types
+	configureClient(config)
+
+	// make a REST client with that config
+	return rest.RESTClientFor(config)
+}
+
+// configureClient sets up a REST client for Fission CRD types.
+//
+// This is copied from the client-go CRD example.  (I don't understand
+// all of it completely.)  It registers our types with the global API
+// "scheme" (api.Scheme), which keeps a directory of types [I guess so
+// it can use the string in the Kind field to make a Go object?].  It
+// also puts the fission CRD types under a "group version" which we
+// create for our CRDs types.
+func configureClient(config *rest.Config) {
+	groupversion := schema.GroupVersion{
+		Group:   "fission.io",
+		Version: "v1",
+	}
+	config.GroupVersion = &groupversion
+	config.APIPath = "/apis"
+	config.ContentType = runtime.ContentTypeJSON
+	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
+
+	schemeBuilder := runtime.NewSchemeBuilder(
+		func(scheme *runtime.Scheme) error {
+			scheme.AddKnownTypes(
+				groupversion,
+				&fv1.Function{},
+				&fv1.FunctionList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
+			scheme.AddKnownTypes(
+				groupversion,
+				&fv1.Environment{},
+				&fv1.EnvironmentList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
+			scheme.AddKnownTypes(
+				groupversion,
+				&fv1.HTTPTrigger{},
+				&fv1.HTTPTriggerList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
+			scheme.AddKnownTypes(
+				groupversion,
+				&fv1.KubernetesWatchTrigger{},
+				&fv1.KubernetesWatchTriggerList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
+			scheme.AddKnownTypes(
+				groupversion,
+				&fv1.TimeTrigger{},
+				&fv1.TimeTriggerList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
+			scheme.AddKnownTypes(
+				groupversion,
+				&fv1.MessageQueueTrigger{},
+				&fv1.MessageQueueTriggerList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
+			scheme.AddKnownTypes(
+				groupversion,
+				&fv1.Package{},
+				&fv1.PackageList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
+			scheme.AddKnownTypes(
+				groupversion,
+				&fv1.Recorder{},
+				&fv1.RecorderList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
+			scheme.AddKnownTypes(
+				groupversion,
+				&fv1.CanaryConfig{},
+				&fv1.CanaryConfigList{},
+				&metav1.ListOptions{},
+				&metav1.DeleteOptions{},
+			)
+			return nil
+		})
+	schemeBuilder.AddToScheme(scheme.Scheme)
+}
+
+func waitForCRDs(crdClient *rest.RESTClient) error {
+	start := time.Now()
+	for {
+		fi := MakeFunctionInterface(crdClient, metav1.NamespaceDefault)
+		_, err := fi.List(metav1.ListOptions{})
+		if err != nil {
+			time.Sleep(100 * time.Millisecond)
+		} else {
+			return nil
+		}
+
+		if time.Since(start) > 30*time.Second {
+			return errors.New("timeout waiting for CRDs")
+		}
+	}
+}
+
+func MakeFissionClient() (*FissionClient, *kubernetes.Clientset, *apiextensionsclient.Clientset, error) {
+	config, kubeClient, apiExtClient, err := GetKubernetesClient()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	crdClient, err := GetCrdClient(config)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	fc := &FissionClient{
+		crdClient: crdClient,
+	}
+	return fc, kubeClient, apiExtClient, nil
+}
+
+func (fc *FissionClient) Functions(ns string) FunctionInterface {
+	return MakeFunctionInterface(fc.crdClient, ns)
+}
+func (fc *FissionClient) Environments(ns string) EnvironmentInterface {
+	return MakeEnvironmentInterface(fc.crdClient, ns)
+}
+func (fc *FissionClient) HTTPTriggers(ns string) HTTPTriggerInterface {
+	return MakeHTTPTriggerInterface(fc.crdClient, ns)
+}
+func (fc *FissionClient) KubernetesWatchTriggers(ns string) KubernetesWatchTriggerInterface {
+	return MakeKubernetesWatchTriggerInterface(fc.crdClient, ns)
+}
+func (fc *FissionClient) TimeTriggers(ns string) TimeTriggerInterface {
+	return MakeTimeTriggerInterface(fc.crdClient, ns)
+}
+func (fc *FissionClient) MessageQueueTriggers(ns string) MessageQueueTriggerInterface {
+	return MakeMessageQueueTriggerInterface(fc.crdClient, ns)
+}
+func (fc *FissionClient) Recorders(ns string) RecorderInterface {
+	return MakeRecorderInterface(fc.crdClient, ns)
+}
+func (fc *FissionClient) Packages(ns string) PackageInterface {
+	return MakePackageInterface(fc.crdClient, ns)
+}
+func (fc *FissionClient) CanaryConfigs(ns string) CanaryConfigInterface {
+	return MakeCanaryConfigInterface(fc.crdClient, ns)
+}
+func (fc *FissionClient) WaitForCRDs() error {
+	return waitForCRDs(fc.crdClient)
+}
+func (fc *FissionClient) GetCrdClient() *rest.RESTClient {
+	return fc.crdClient
+}

pkg/crd/crd.go

@@ -0,0 +1,216 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	"time"
+
+	"go.uber.org/zap"
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	crdGroupName = "fission.io"
+	crdVersion   = "v1"
+)
+
+// ensureCRD checks if the given CRD type exists, and creates it if
+// needed. (Note that this creates the CRD type; it doesn't create any
+// _instances_ of that type.)
+func ensureCRD(logger *zap.Logger, clientset *apiextensionsclient.Clientset, crd *apiextensionsv1beta1.CustomResourceDefinition) (err error) {
+	maxRetries := 5
+
+	for i := 0; i < maxRetries; i++ {
+		_, err = clientset.ApiextensionsV1beta1().CustomResourceDefinitions().Create(crd)
+		if err == nil {
+			return nil
+		}
+
+		// return if the resource already exists
+		if k8serrors.IsAlreadyExists(err) {
+			return nil
+		} else {
+			// The requests fail to connect to k8s api server before
+			// istio-prxoy is ready to serve traffic. Retry again.
+			logger.Info("error connecting to kubernetes api service, retrying", zap.Error(err))
+			time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
+			continue
+		}
+	}
+
+	return err
+}
+
+// Ensure CRDs
+func EnsureFissionCRDs(logger *zap.Logger, clientset *apiextensionsclient.Clientset) error {
+	crds := []apiextensionsv1beta1.CustomResourceDefinition{
+		// Functions
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "functions.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "Function",
+					Plural:   "functions",
+					Singular: "function",
+				},
+			},
+		},
+		// Environments (function containers)
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "environments.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "Environment",
+					Plural:   "environments",
+					Singular: "environment",
+				},
+			},
+		},
+		// HTTP triggers for functions
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "httptriggers.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "HTTPTrigger",
+					Plural:   "httptriggers",
+					Singular: "httptrigger",
+				},
+			},
+		},
+		// Kubernetes watch triggers for functions
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "kuberneteswatchtriggers.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "KubernetesWatchTrigger",
+					Plural:   "kuberneteswatchtriggers",
+					Singular: "kuberneteswatchtrigger",
+				},
+			},
+		},
+		// Time-based triggers for functions
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "timetriggers.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "TimeTrigger",
+					Plural:   "timetriggers",
+					Singular: "timetrigger",
+				},
+			},
+		},
+		// Message queue triggers for functions
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "messagequeuetriggers.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "MessageQueueTrigger",
+					Plural:   "messagequeuetriggers",
+					Singular: "messagequeuetrigger",
+				},
+			},
+		},
+		// Recorders
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "recorders.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "Recorder",
+					Plural:   "recorders",
+					Singular: "recorder",
+				},
+			},
+		},
+		// Packages: archives containing source or binaries for one or more functions
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "packages.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "Package",
+					Plural:   "packages",
+					Singular: "package",
+				},
+			},
+		},
+		// CanaryConfig: configuration for canary deployment of functions
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "canaryconfigs.fission.io",
+			},
+			Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
+				Group:   crdGroupName,
+				Version: crdVersion,
+				Scope:   apiextensionsv1beta1.NamespaceScoped,
+				Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
+					Kind:     "CanaryConfig",
+					Plural:   "canaryconfigs",
+					Singular: "canaryconfig",
+				},
+			},
+		},
+	}
+	for _, crd := range crds {
+		err := ensureCRD(logger, clientset, &crd)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/crd/crd_test.go

@@ -0,0 +1,449 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func panicIf(err error) {
+	if err != nil {
+		log.Panicf("err: %v", err)
+	}
+}
+
+func functionTests(crdClient *rest.RESTClient) {
+	// sample function object
+	function := &fv1.Function{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Function",
+			APIVersion: "fv1.io/v1",
+		},
+		Metadata: metav1.ObjectMeta{
+			Name:      "hello",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: fv1.FunctionSpec{
+			Package: fv1.FunctionPackageRef{
+				PackageRef: fv1.PackageRef{
+					Name:      "foo",
+					Namespace: "bar",
+				},
+				FunctionName: "hello",
+			},
+			Environment: fv1.EnvironmentReference{
+				Name: "xxx",
+			},
+		},
+	}
+
+	// Test function CRUD
+	fi := MakeFunctionInterface(crdClient, metav1.NamespaceDefault)
+
+	// cleanup from old crashed tests, ignore errors
+	fi.Delete(function.Metadata.Name, nil)
+
+	// create
+	f, err := fi.Create(function)
+	panicIf(err)
+	if f.Metadata.Name != function.Metadata.Name {
+		log.Panicf("Bad result from create: %v", f)
+	}
+
+	// read
+	f, err = fi.Get(function.Metadata.Name)
+	panicIf(err)
+	if f.Spec.Environment.Name != function.Spec.Environment.Name {
+		log.Panicf("Bad result from Get: %v", f)
+	}
+
+	log.Printf("f.Metadata = %#v", f.Metadata)
+
+	// update
+	function.Metadata.ResourceVersion = f.Metadata.ResourceVersion
+	function.Spec.Environment.Name = "yyy"
+	f, err = fi.Update(function)
+	panicIf(err)
+
+	log.Printf("f.Metadata = %#v", f.Metadata)
+
+	// list
+	fl, err := fi.List(metav1.ListOptions{})
+	panicIf(err)
+	if len(fl.Items) != 1 {
+		log.Panicf("wrong count from function list: %v", len(fl.Items))
+	}
+	if fl.Items[0].Spec.Environment.Name != f.Spec.Environment.Name {
+		log.Panicf("bad object from list: %v", fl.Items[0])
+	}
+
+	// delete
+	err = fi.Delete(f.Metadata.Name, nil)
+	panicIf(err)
+
+	// start a watch
+	wi, err := fi.Watch(metav1.ListOptions{})
+	panicIf(err)
+
+	start := time.Now()
+	function.Metadata.ResourceVersion = ""
+	f, err = fi.Create(function)
+	panicIf(err)
+	defer fi.Delete(f.Metadata.Name, nil)
+
+	// assert that we get a watch event for the new function
+	recvd := false
+	select {
+	case <-time.NewTimer(1 * time.Second).C:
+		if !recvd {
+			log.Panicf("Didn't get watch event")
+		}
+	case ev := <-wi.ResultChan():
+		wf, ok := ev.Object.(*fv1.Function)
+		if !ok {
+			log.Panicf("Can't cast to Function")
+		}
+		if wf.Spec.Environment.Name != function.Spec.Environment.Name {
+			log.Panicf("Bad object from watch: %#v", wf)
+		}
+		log.Printf("watch event took %v", time.Since(start))
+		recvd = true
+	}
+
+}
+
+func environmentTests(crdClient *rest.RESTClient) {
+	// sample environment object
+	environment := &fv1.Environment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Environment",
+			APIVersion: "fv1.io/v1",
+		},
+		Metadata: metav1.ObjectMeta{
+			Name:      "hello",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: fv1.EnvironmentSpec{
+			Runtime: fv1.Runtime{
+				Image: "xxx",
+			},
+			Builder: fv1.Builder{
+				Image:   "yyy",
+				Command: "zzz",
+			},
+		},
+	}
+
+	// Test environment CRUD
+	ei := MakeEnvironmentInterface(crdClient, metav1.NamespaceDefault)
+
+	// cleanup from old crashed tests, ignore errors
+	ei.Delete(environment.Metadata.Name, nil)
+
+	// create
+	e, err := ei.Create(environment)
+	panicIf(err)
+	if e.Metadata.Name != environment.Metadata.Name {
+		log.Panicf("Bad result from create: %v", e)
+	}
+
+	// read
+	e, err = ei.Get(environment.Metadata.Name)
+	panicIf(err)
+	if len(e.Spec.Runtime.Image) != len(environment.Spec.Runtime.Image) {
+		log.Panicf("Bad result from Get: %#v", e)
+	}
+
+	// update
+	environment.Metadata.ResourceVersion = e.Metadata.ResourceVersion
+	environment.Spec.Runtime.Image = "www"
+	e, err = ei.Update(environment)
+	panicIf(err)
+
+	// list
+	el, err := ei.List(metav1.ListOptions{})
+	panicIf(err)
+	if len(el.Items) != 1 {
+		log.Panicf("wrong count from environment list: %v", len(el.Items))
+	}
+	if el.Items[0].Spec.Runtime.Image != e.Spec.Runtime.Image {
+		log.Panicf("bad object from list: %v", el.Items[0])
+	}
+
+	// delete
+	err = ei.Delete(e.Metadata.Name, nil)
+	panicIf(err)
+
+	// start a watch
+	wi, err := ei.Watch(metav1.ListOptions{})
+	panicIf(err)
+
+	start := time.Now()
+	environment.Metadata.ResourceVersion = ""
+	e, err = ei.Create(environment)
+	panicIf(err)
+	defer ei.Delete(e.Metadata.Name, nil)
+
+	// assert that we get a watch event for the new environment
+	recvd := false
+	select {
+	case <-time.NewTimer(1 * time.Second).C:
+		if !recvd {
+			log.Panicf("Didn't get watch event")
+		}
+	case ev := <-wi.ResultChan():
+		obj, ok := ev.Object.(*fv1.Environment)
+		if !ok {
+			log.Panicf("Can't cast to Environment")
+		}
+		if obj.Spec.Runtime.Image != environment.Spec.Runtime.Image {
+			log.Panicf("Bad object from watch: %#v", obj)
+		}
+		log.Printf("watch event took %v", time.Since(start))
+		recvd = true
+	}
+
+}
+
+func httpTriggerTests(crdClient *rest.RESTClient) {
+	// sample httpTrigger object
+	httpTrigger := &fv1.HTTPTrigger{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "HTTPTrigger",
+			APIVersion: "fv1.io/v1",
+		},
+		Metadata: metav1.ObjectMeta{
+			Name:      "hello",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: fv1.HTTPTriggerSpec{
+			RelativeURL: "/hi",
+			Method:      "GET",
+			FunctionReference: fv1.FunctionReference{
+				Type: fv1.FunctionReferenceTypeFunctionName,
+				Name: "hello",
+			},
+		},
+	}
+
+	// Test httpTrigger CRUD
+	ei := MakeHTTPTriggerInterface(crdClient, metav1.NamespaceDefault)
+
+	// cleanup from old crashed tests, ignore errors
+	ei.Delete(httpTrigger.Metadata.Name, nil)
+
+	// create
+	e, err := ei.Create(httpTrigger)
+	panicIf(err)
+	if e.Metadata.Name != httpTrigger.Metadata.Name {
+		log.Panicf("Bad result from create: %v", e)
+	}
+
+	// read
+	e, err = ei.Get(httpTrigger.Metadata.Name)
+	panicIf(err)
+	if len(e.Spec.Method) != len(httpTrigger.Spec.Method) {
+		log.Panicf("Bad result from Get: %#v", e)
+	}
+
+	// update
+	httpTrigger.Metadata.ResourceVersion = e.Metadata.ResourceVersion
+	httpTrigger.Spec.Method = "POST"
+	e, err = ei.Update(httpTrigger)
+	panicIf(err)
+
+	// list
+	el, err := ei.List(metav1.ListOptions{})
+	panicIf(err)
+	if len(el.Items) != 1 {
+		log.Panicf("wrong count from http trigger list: %v", len(el.Items))
+	}
+	if el.Items[0].Spec.Method != e.Spec.Method {
+		log.Panicf("bad object from list: %v", el.Items[0])
+	}
+
+	// delete
+	err = ei.Delete(e.Metadata.Name, nil)
+	panicIf(err)
+
+	// start a watch
+	wi, err := ei.Watch(metav1.ListOptions{})
+	panicIf(err)
+
+	start := time.Now()
+	httpTrigger.Metadata.ResourceVersion = ""
+	e, err = ei.Create(httpTrigger)
+	panicIf(err)
+	defer ei.Delete(e.Metadata.Name, nil)
+
+	// assert that we get a watch event for the new httpTrigger
+	recvd := false
+	select {
+	case <-time.NewTimer(1 * time.Second).C:
+		if !recvd {
+			log.Panicf("Didn't get watch event")
+		}
+	case ev := <-wi.ResultChan():
+		obj, ok := ev.Object.(*fv1.HTTPTrigger)
+		if !ok {
+			log.Panicf("Can't cast to HTTPTrigger")
+		}
+		if obj.Spec.Method != httpTrigger.Spec.Method {
+			log.Panicf("Bad object from watch: %#v", obj)
+		}
+		log.Printf("watch event took %v", time.Since(start))
+		recvd = true
+	}
+
+}
+
+func kubernetesWatchTriggerTests(crdClient *rest.RESTClient) {
+	// sample kubernetesWatchTrigger object
+	kubernetesWatchTrigger := &fv1.KubernetesWatchTrigger{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "KubernetesWatchTrigger",
+			APIVersion: "fv1.io/v1",
+		},
+		Metadata: metav1.ObjectMeta{
+			Name:      "hello",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: fv1.KubernetesWatchTriggerSpec{
+			Namespace: "foo",
+			Type:      "pod",
+			LabelSelector: map[string]string{
+				"x": "y",
+			},
+			FunctionReference: fv1.FunctionReference{
+				Type: fv1.FunctionReferenceTypeFunctionName,
+				Name: "foo",
+			},
+		},
+	}
+
+	// Test kubernetesWatchTrigger CRUD
+	ei := MakeKubernetesWatchTriggerInterface(crdClient, metav1.NamespaceDefault)
+
+	// cleanup from old crashed tests, ignore errors
+	ei.Delete(kubernetesWatchTrigger.Metadata.Name, nil)
+
+	// create
+	e, err := ei.Create(kubernetesWatchTrigger)
+	panicIf(err)
+	if e.Metadata.Name != kubernetesWatchTrigger.Metadata.Name {
+		log.Panicf("Bad result from create: %v", e)
+	}
+
+	// read
+	e, err = ei.Get(kubernetesWatchTrigger.Metadata.Name)
+	panicIf(err)
+	if e.Spec.Type != kubernetesWatchTrigger.Spec.Type {
+		log.Panicf("Bad result from Get: %#v", e)
+	}
+
+	// update
+	kubernetesWatchTrigger.Metadata.ResourceVersion = e.Metadata.ResourceVersion
+	kubernetesWatchTrigger.Spec.Type = "service"
+	e, err = ei.Update(kubernetesWatchTrigger)
+	panicIf(err)
+
+	// list
+	el, err := ei.List(metav1.ListOptions{})
+	panicIf(err)
+	if len(el.Items) != 1 {
+		log.Panicf("wrong count from kubeWatcher list: %v", len(el.Items))
+	}
+	if el.Items[0].Spec.Type != e.Spec.Type {
+		log.Panicf("bad object from list: %v", el.Items[0])
+	}
+
+	// delete
+	err = ei.Delete(e.Metadata.Name, nil)
+	panicIf(err)
+
+	// start a watch
+	wi, err := ei.Watch(metav1.ListOptions{})
+	panicIf(err)
+
+	start := time.Now()
+	kubernetesWatchTrigger.Metadata.ResourceVersion = ""
+	e, err = ei.Create(kubernetesWatchTrigger)
+	panicIf(err)
+	defer ei.Delete(e.Metadata.Name, nil)
+
+	// assert that we get a watch event for the new kubernetesWatchTrigger
+	recvd := false
+	select {
+	case <-time.NewTimer(1 * time.Second).C:
+		if !recvd {
+			log.Panicf("Didn't get watch event")
+		}
+	case ev := <-wi.ResultChan():
+		obj, ok := ev.Object.(*fv1.KubernetesWatchTrigger)
+		if !ok {
+			log.Panicf("Can't cast to KubernetesWatchTrigger")
+		}
+		if obj.Spec.Type != kubernetesWatchTrigger.Spec.Type {
+			log.Panicf("Bad object from watch: %#v", obj)
+		}
+		log.Printf("watch event took %v", time.Since(start))
+		recvd = true
+	}
+
+}
+
+func TestCrd(t *testing.T) {
+	// skip test if no cluster available for testing
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if len(kubeconfig) == 0 {
+		log.Println("Skipping test, no kubernetes cluster")
+		return
+	}
+
+	// Create the client config. Needs the KUBECONFIG env var to
+	// point at a valid kubeconfig.
+	config, _, apiExtClient, err := GetKubernetesClient()
+	panicIf(err)
+
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	// init our types
+	err = EnsureFissionCRDs(logger, apiExtClient)
+	panicIf(err)
+
+	// rest client with knowledge about our crd types
+	crdClient, err := GetCrdClient(config)
+	panicIf(err)
+
+	err = waitForCRDs(crdClient)
+	panicIf(err)
+
+	functionTests(crdClient)
+	environmentTests(crdClient)
+	httpTriggerTests(crdClient)
+	kubernetesWatchTriggerTests(crdClient)
+}

pkg/crd/environment.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	EnvironmentInterface interface {
+		Create(*fv1.Environment) (*fv1.Environment, error)
+		Get(name string) (*fv1.Environment, error)
+		Update(*fv1.Environment) (*fv1.Environment, error)
+		Delete(name string, options *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*fv1.EnvironmentList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	environmentClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakeEnvironmentInterface(crdClient *rest.RESTClient, namespace string) EnvironmentInterface {
+	return &environmentClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (ec *environmentClient) Create(e *fv1.Environment) (*fv1.Environment, error) {
+	var result fv1.Environment
+	err := ec.client.Post().
+		Resource("environments").
+		Namespace(ec.namespace).
+		Body(e).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (ec *environmentClient) Get(name string) (*fv1.Environment, error) {
+	var result fv1.Environment
+	err := ec.client.Get().
+		Resource("environments").
+		Namespace(ec.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (ec *environmentClient) Update(e *fv1.Environment) (*fv1.Environment, error) {
+	var result fv1.Environment
+	err := ec.client.Put().
+		Resource("environments").
+		Namespace(ec.namespace).
+		Name(e.Metadata.Name).
+		Body(e).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (ec *environmentClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return ec.client.Delete().
+		Namespace(ec.namespace).
+		Resource("environments").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (ec *environmentClient) List(opts metav1.ListOptions) (*fv1.EnvironmentList, error) {
+	var result fv1.EnvironmentList
+	err := ec.client.Get().
+		Namespace(ec.namespace).
+		Resource("environments").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (ec *environmentClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return ec.client.Get().
+		Prefix("watch").
+		Namespace(ec.namespace).
+		Resource("environments").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

pkg/crd/function.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	FunctionInterface interface {
+		Create(*fv1.Function) (*fv1.Function, error)
+		Get(name string) (*fv1.Function, error)
+		Update(*fv1.Function) (*fv1.Function, error)
+		Delete(name string, options *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*fv1.FunctionList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	functionClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakeFunctionInterface(crdClient *rest.RESTClient, namespace string) FunctionInterface {
+	return &functionClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (fc *functionClient) Create(f *fv1.Function) (*fv1.Function, error) {
+	var result fv1.Function
+	err := fc.client.Post().
+		Resource("functions").
+		Namespace(fc.namespace).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *functionClient) Get(name string) (*fv1.Function, error) {
+	var result fv1.Function
+	err := fc.client.Get().
+		Resource("functions").
+		Namespace(fc.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *functionClient) Update(f *fv1.Function) (*fv1.Function, error) {
+	var result fv1.Function
+	err := fc.client.Put().
+		Resource("functions").
+		Namespace(fc.namespace).
+		Name(f.Metadata.Name).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *functionClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return fc.client.Delete().
+		Namespace(fc.namespace).
+		Resource("functions").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (fc *functionClient) List(opts metav1.ListOptions) (*fv1.FunctionList, error) {
+	var result fv1.FunctionList
+	err := fc.client.Get().
+		Namespace(fc.namespace).
+		Resource("functions").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *functionClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return fc.client.Get().
+		Prefix("watch").
+		Namespace(fc.namespace).
+		Resource("functions").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

pkg/crd/httptrigger.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	HTTPTriggerInterface interface {
+		Create(*fv1.HTTPTrigger) (*fv1.HTTPTrigger, error)
+		Get(name string) (*fv1.HTTPTrigger, error)
+		Update(*fv1.HTTPTrigger) (*fv1.HTTPTrigger, error)
+		Delete(name string, options *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*fv1.HTTPTriggerList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	httpTriggerClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakeHTTPTriggerInterface(crdClient *rest.RESTClient, namespace string) HTTPTriggerInterface {
+	return &httpTriggerClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (c *httpTriggerClient) Create(obj *fv1.HTTPTrigger) (*fv1.HTTPTrigger, error) {
+	var result fv1.HTTPTrigger
+	err := c.client.Post().
+		Resource("httptriggers").
+		Namespace(c.namespace).
+		Body(obj).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *httpTriggerClient) Get(name string) (*fv1.HTTPTrigger, error) {
+	var result fv1.HTTPTrigger
+	err := c.client.Get().
+		Resource("httptriggers").
+		Namespace(c.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *httpTriggerClient) Update(obj *fv1.HTTPTrigger) (*fv1.HTTPTrigger, error) {
+	var result fv1.HTTPTrigger
+	err := c.client.Put().
+		Resource("httptriggers").
+		Namespace(c.namespace).
+		Name(obj.Metadata.Name).
+		Body(obj).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *httpTriggerClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return c.client.Delete().
+		Namespace(c.namespace).
+		Resource("httptriggers").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (c *httpTriggerClient) List(opts metav1.ListOptions) (*fv1.HTTPTriggerList, error) {
+	var result fv1.HTTPTriggerList
+	err := c.client.Get().
+		Namespace(c.namespace).
+		Resource("httptriggers").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *httpTriggerClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return c.client.Get().
+		Prefix("watch").
+		Namespace(c.namespace).
+		Resource("httptriggers").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

pkg/crd/key.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// Given metadata, create a key that uniquely identifies the contents
+// of the object. Since resourceVersion changes on every update and
+// UIDs are unique, uid+resourceVersion identifies the
+// content. (ResourceVersion may also update on status updates, so
+// this will result in some unnecessary cache misses. That should be
+// ok.)
+func CacheKey(metadata *metav1.ObjectMeta) string {
+	return fmt.Sprintf("%v_%v", metadata.UID, metadata.ResourceVersion)
+}

pkg/crd/kubernetesWatchTrigger.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	KubernetesWatchTriggerInterface interface {
+		Create(*fv1.KubernetesWatchTrigger) (*fv1.KubernetesWatchTrigger, error)
+		Get(name string) (*fv1.KubernetesWatchTrigger, error)
+		Update(*fv1.KubernetesWatchTrigger) (*fv1.KubernetesWatchTrigger, error)
+		Delete(name string, options *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*fv1.KubernetesWatchTriggerList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	kubernetesWatchTriggerClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakeKubernetesWatchTriggerInterface(crdClient *rest.RESTClient, namespace string) KubernetesWatchTriggerInterface {
+	return &kubernetesWatchTriggerClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (c *kubernetesWatchTriggerClient) Create(obj *fv1.KubernetesWatchTrigger) (*fv1.KubernetesWatchTrigger, error) {
+	var result fv1.KubernetesWatchTrigger
+	err := c.client.Post().
+		Resource("kuberneteswatchtriggers").
+		Namespace(c.namespace).
+		Body(obj).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *kubernetesWatchTriggerClient) Get(name string) (*fv1.KubernetesWatchTrigger, error) {
+	var result fv1.KubernetesWatchTrigger
+	err := c.client.Get().
+		Resource("kuberneteswatchtriggers").
+		Namespace(c.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *kubernetesWatchTriggerClient) Update(obj *fv1.KubernetesWatchTrigger) (*fv1.KubernetesWatchTrigger, error) {
+	var result fv1.KubernetesWatchTrigger
+	err := c.client.Put().
+		Resource("kuberneteswatchtriggers").
+		Namespace(c.namespace).
+		Name(obj.Metadata.Name).
+		Body(obj).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *kubernetesWatchTriggerClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return c.client.Delete().
+		Namespace(c.namespace).
+		Resource("kuberneteswatchtriggers").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (c *kubernetesWatchTriggerClient) List(opts metav1.ListOptions) (*fv1.KubernetesWatchTriggerList, error) {
+	var result fv1.KubernetesWatchTriggerList
+	err := c.client.Get().
+		Namespace(c.namespace).
+		Resource("kuberneteswatchtriggers").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *kubernetesWatchTriggerClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return c.client.Get().
+		Prefix("watch").
+		Namespace(c.namespace).
+		Resource("kuberneteswatchtriggers").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

pkg/crd/messagequeuetrigger.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	MessageQueueTriggerInterface interface {
+		Create(*fv1.MessageQueueTrigger) (*fv1.MessageQueueTrigger, error)
+		Get(name string) (*fv1.MessageQueueTrigger, error)
+		Update(*fv1.MessageQueueTrigger) (*fv1.MessageQueueTrigger, error)
+		Delete(name string, options *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*fv1.MessageQueueTriggerList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	messageQueueTriggerClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakeMessageQueueTriggerInterface(crdClient *rest.RESTClient, namespace string) MessageQueueTriggerInterface {
+	return &messageQueueTriggerClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (fc *messageQueueTriggerClient) Create(f *fv1.MessageQueueTrigger) (*fv1.MessageQueueTrigger, error) {
+	var result fv1.MessageQueueTrigger
+	err := fc.client.Post().
+		Resource("messagequeuetriggers").
+		Namespace(fc.namespace).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *messageQueueTriggerClient) Get(name string) (*fv1.MessageQueueTrigger, error) {
+	var result fv1.MessageQueueTrigger
+	err := fc.client.Get().
+		Resource("messagequeuetriggers").
+		Namespace(fc.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *messageQueueTriggerClient) Update(f *fv1.MessageQueueTrigger) (*fv1.MessageQueueTrigger, error) {
+	var result fv1.MessageQueueTrigger
+	err := fc.client.Put().
+		Resource("messagequeuetriggers").
+		Namespace(fc.namespace).
+		Name(f.Metadata.Name).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *messageQueueTriggerClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return fc.client.Delete().
+		Namespace(fc.namespace).
+		Resource("messagequeuetriggers").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (fc *messageQueueTriggerClient) List(opts metav1.ListOptions) (*fv1.MessageQueueTriggerList, error) {
+	var result fv1.MessageQueueTriggerList
+	err := fc.client.Get().
+		Namespace(fc.namespace).
+		Resource("messagequeuetriggers").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *messageQueueTriggerClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return fc.client.Get().
+		Prefix("watch").
+		Namespace(fc.namespace).
+		Resource("messagequeuetriggers").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

pkg/crd/package.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	PackageInterface interface {
+		Create(*fv1.Package) (*fv1.Package, error)
+		Get(name string) (*fv1.Package, error)
+		Update(*fv1.Package) (*fv1.Package, error)
+		Delete(name string, options *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*fv1.PackageList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	packageClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakePackageInterface(crdClient *rest.RESTClient, namespace string) PackageInterface {
+	return &packageClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (c *packageClient) Create(f *fv1.Package) (*fv1.Package, error) {
+	var result fv1.Package
+	err := c.client.Post().
+		Resource("packages").
+		Namespace(c.namespace).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *packageClient) Get(name string) (*fv1.Package, error) {
+	var result fv1.Package
+	err := c.client.Get().
+		Resource("packages").
+		Namespace(c.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *packageClient) Update(f *fv1.Package) (*fv1.Package, error) {
+	var result fv1.Package
+	err := c.client.Put().
+		Resource("packages").
+		Namespace(c.namespace).
+		Name(f.Metadata.Name).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *packageClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return c.client.Delete().
+		Namespace(c.namespace).
+		Resource("packages").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (c *packageClient) List(opts metav1.ListOptions) (*fv1.PackageList, error) {
+	var result fv1.PackageList
+	err := c.client.Get().
+		Namespace(c.namespace).
+		Resource("packages").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (c *packageClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return c.client.Get().
+		Prefix("watch").
+		Namespace(c.namespace).
+		Resource("packages").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

pkg/crd/recorder.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	RecorderInterface interface {
+		Create(*fv1.Recorder) (*fv1.Recorder, error)
+		Get(name string) (*fv1.Recorder, error)
+		Update(*fv1.Recorder) (*fv1.Recorder, error)
+		Delete(name string, opts *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*fv1.RecorderList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	recorderClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakeRecorderInterface(crdClient *rest.RESTClient, namespace string) RecorderInterface {
+	return &recorderClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (rc *recorderClient) Create(r *fv1.Recorder) (*fv1.Recorder, error) {
+	var result fv1.Recorder
+	err := rc.client.Post().
+		Resource("recorders").
+		Namespace("default").
+		Body(r).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (rc *recorderClient) Get(name string) (*fv1.Recorder, error) {
+	var result fv1.Recorder
+	err := rc.client.Get().
+		Resource("recorders").
+		Namespace(rc.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (rc *recorderClient) Update(r *fv1.Recorder) (*fv1.Recorder, error) {
+	var result fv1.Recorder
+	err := rc.client.Put().
+		Resource("recorders").
+		Namespace(rc.namespace).
+		Name(r.Metadata.Name).
+		Body(r).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (rc *recorderClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return rc.client.Delete().
+		Namespace(rc.namespace).
+		Resource("recorders").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (rc *recorderClient) List(opts metav1.ListOptions) (*fv1.RecorderList, error) {
+	var result fv1.RecorderList
+	err := rc.client.Get().
+		Namespace(rc.namespace).
+		Resource("recorders").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (rc *recorderClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return rc.client.Get().
+		Prefix("watch").
+		Namespace(rc.namespace).
+		Resource("recorders").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

pkg/crd/timetrigger.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type (
+	TimeTriggerInterface interface {
+		Create(*fv1.TimeTrigger) (*fv1.TimeTrigger, error)
+		Get(name string) (*fv1.TimeTrigger, error)
+		Update(*fv1.TimeTrigger) (*fv1.TimeTrigger, error)
+		Delete(name string, options *metav1.DeleteOptions) error
+		List(opts metav1.ListOptions) (*fv1.TimeTriggerList, error)
+		Watch(opts metav1.ListOptions) (watch.Interface, error)
+	}
+
+	timeTriggerClient struct {
+		client    *rest.RESTClient
+		namespace string
+	}
+)
+
+func MakeTimeTriggerInterface(crdClient *rest.RESTClient, namespace string) TimeTriggerInterface {
+	return &timeTriggerClient{
+		client:    crdClient,
+		namespace: namespace,
+	}
+}
+
+func (fc *timeTriggerClient) Create(f *fv1.TimeTrigger) (*fv1.TimeTrigger, error) {
+	var result fv1.TimeTrigger
+	err := fc.client.Post().
+		Resource("timetriggers").
+		Namespace(fc.namespace).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *timeTriggerClient) Get(name string) (*fv1.TimeTrigger, error) {
+	var result fv1.TimeTrigger
+	err := fc.client.Get().
+		Resource("timetriggers").
+		Namespace(fc.namespace).
+		Name(name).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *timeTriggerClient) Update(f *fv1.TimeTrigger) (*fv1.TimeTrigger, error) {
+	var result fv1.TimeTrigger
+	err := fc.client.Put().
+		Resource("timetriggers").
+		Namespace(fc.namespace).
+		Name(f.Metadata.Name).
+		Body(f).
+		Do().Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *timeTriggerClient) Delete(name string, opts *metav1.DeleteOptions) error {
+	return fc.client.Delete().
+		Namespace(fc.namespace).
+		Resource("timetriggers").
+		Name(name).
+		Body(opts).
+		Do().
+		Error()
+}
+
+func (fc *timeTriggerClient) List(opts metav1.ListOptions) (*fv1.TimeTriggerList, error) {
+	var result fv1.TimeTriggerList
+	err := fc.client.Get().
+		Namespace(fc.namespace).
+		Resource("timetriggers").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Do().
+		Into(&result)
+	if err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+func (fc *timeTriggerClient) Watch(opts metav1.ListOptions) (watch.Interface, error) {
+	return fc.client.Get().
+		Prefix("watch").
+		Namespace(fc.namespace).
+		Resource("timetriggers").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Watch()
+}

pkg/error/httperror.go

@@ -0,0 +1,136 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package error
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+type (
+	// Errors returned by the Fission API.
+	Error struct {
+		Code    errorCode `json:"code"`
+		Message string    `json:"message"`
+	}
+
+	errorCode int
+)
+
+func (err Error) Error() string {
+	return fmt.Sprintf("%v - %v", err.Description(), err.Message)
+}
+
+func MakeError(code int, msg string) Error {
+	return Error{Code: errorCode(code), Message: msg}
+}
+
+func MakeErrorFromHTTP(resp *http.Response) error {
+	if resp.StatusCode == http.StatusOK {
+		return nil
+	}
+
+	var errCode int
+	switch resp.StatusCode {
+	case http.StatusBadRequest:
+		errCode = ErrorInvalidArgument
+	case http.StatusForbidden:
+		errCode = ErrorNotAuthorized
+	case http.StatusNotFound:
+		errCode = ErrorNotFound
+	case http.StatusConflict:
+		errCode = ErrorNameExists
+	default:
+		errCode = ErrorInternal
+	}
+
+	msg := resp.Status
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err == nil && len(body) > 0 {
+		msg = strings.TrimSpace(string(body))
+	}
+
+	return MakeError(errCode, msg)
+}
+
+func (err Error) HTTPStatus() int {
+	var code int
+	switch err.Code {
+	case ErrorInvalidArgument:
+		code = http.StatusBadRequest
+	case ErrorNotAuthorized:
+		code = http.StatusForbidden
+	case ErrorNotFound:
+		code = http.StatusNotFound
+	case ErrorNameExists:
+		code = http.StatusConflict
+	default:
+		code = http.StatusInternalServerError
+	}
+	return code
+}
+
+func GetHTTPError(err error) (int, string) {
+	var msg string
+	var code int
+	fe, ok := err.(Error)
+	if ok {
+		code = fe.HTTPStatus()
+		msg = fe.Message
+	} else {
+		code = http.StatusInternalServerError
+		msg = err.Error()
+	}
+	return code, msg
+}
+
+func (err Error) Description() string {
+	idx := int(err.Code)
+	if idx < 0 || idx > len(errorDescriptions)-1 {
+		return ""
+	}
+	return errorDescriptions[idx]
+}
+
+const (
+	ErrorInternal = iota
+
+	ErrorNotAuthorized
+	ErrorNotFound
+	ErrorNameExists
+	ErrorInvalidArgument
+	ErrorNoSpace
+	ErrorNotImplmented
+	ErrorChecksumFail
+	ErrorSizeLimitExceeded
+)
+
+// must match order and len of the above const
+var errorDescriptions = []string{
+	"Internal error",
+	"Not authorized",
+	"Resource not found",
+	"Resource exists",
+	"Invalid argument",
+	"No space",
+	"Not implemented",
+	"Checksum verification failed",
+	"Size limit exceeded",
+}

pkg/executor/api.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package executor
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/fission/fission/pkg/utils"
+	"github.com/gorilla/mux"
+	"go.opencensus.io/plugin/ochttp"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+func (executor *Executor) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, "Failed to read request", http.StatusInternalServerError)
+		return
+	}
+
+	// get function metadata
+	m := metav1.ObjectMeta{}
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		http.Error(w, "Failed to parse request", http.StatusBadRequest)
+		return
+	}
+
+	serviceName, err := executor.getServiceForFunction(r.Context(), &m)
+	if err != nil {
+		code, msg := ferror.GetHTTPError(err)
+		executor.logger.Error("error getting service for function",
+			zap.Error(err),
+			zap.String("function", m.Name),
+			zap.String("fission_http_error", msg))
+		http.Error(w, msg, code)
+		return
+	}
+
+	w.Write([]byte(serviceName))
+}
+
+// getServiceForFunction first checks if this function's service is cached, if yes, it validates the address.
+// if it's a valid address, just returns it.
+// else, invalidates its cache entry and makes a new request to create a service for this function and finally responds
+// with new address or error.
+//
+// checking for the validity of the address causes a little more over-head than desired. but, it ensures that
+// stale addresses are not returned to the router.
+// To make it optimal, plan is to add an eager cache invalidator function that watches for pod deletion events and
+// invalidates the cache entry if the pod address was cached.
+func (executor *Executor) getServiceForFunction(ctx context.Context, m *metav1.ObjectMeta) (string, error) {
+	// Check function -> svc cache
+	executor.logger.Info("checking for cached function service",
+		zap.String("function_name", m.Name),
+		zap.String("function_namespace", m.Namespace))
+	fsvc, err := executor.fsCache.GetByFunction(m)
+	if err == nil {
+		if executor.isValidAddress(fsvc) {
+			// Cached, return svc address
+			return fsvc.Address, nil
+		} else {
+			executor.logger.Info("deleting cache entry for invalid address",
+				zap.String("function_name", m.Name),
+				zap.String("function_namespace", m.Namespace),
+				zap.String("address", fsvc.Address))
+			executor.fsCache.DeleteEntry(fsvc)
+		}
+	}
+
+	respChan := make(chan *createFuncServiceResponse)
+	executor.requestChan <- &createFuncServiceRequest{
+		ctx:      ctx,
+		funcMeta: m,
+		respChan: respChan,
+	}
+	resp := <-respChan
+	if resp.err != nil {
+		return "", resp.err
+	}
+	return resp.funcSvc.Address, resp.err
+}
+
+// find funcSvc and update its atime
+func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		executor.logger.Error("failed to read tap service request", zap.Error(err))
+		http.Error(w, "Failed to read request", http.StatusInternalServerError)
+		return
+	}
+	svcName := string(body)
+	svcHost := strings.TrimPrefix(svcName, "http://")
+
+	err = executor.fsCache.TouchByAddress(svcHost)
+	if err != nil {
+		executor.logger.Error("error tapping function service",
+			zap.Error(err),
+			zap.String("service", svcName),
+			zap.String("host", svcHost))
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+func (executor *Executor) healthHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+func (executor *Executor) Serve(port int) {
+	r := mux.NewRouter()
+	r.HandleFunc("/v2/getServiceForFunction", executor.getServiceForFunctionApi).Methods("POST")
+	r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST")
+	r.HandleFunc("/healthz", executor.healthHandler).Methods("GET")
+	address := fmt.Sprintf(":%v", port)
+	executor.logger.Info("starting executor", zap.Int("port", port))
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	executor.ndm.Run(ctx)
+	executor.gpm.Run(ctx)
+	r.Use(utils.LoggingMiddleware(executor.logger))
+	err := http.ListenAndServe(address, &ochttp.Handler{
+		Handler: r,
+		// Propagation: &b3.HTTPFormat{},
+	})
+	executor.logger.Fatal("done listening", zap.Error(err))
+}

pkg/executor/client/client.go

@@ -0,0 +1,126 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"go.opencensus.io/plugin/ochttp"
+	"go.uber.org/zap"
+	"golang.org/x/net/context/ctxhttp"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+type Client struct {
+	logger      *zap.Logger
+	executorUrl string
+	tappedByUrl map[string]bool
+	requestChan chan string
+	httpClient  *http.Client
+}
+
+func MakeClient(logger *zap.Logger, executorUrl string) *Client {
+	c := &Client{
+		logger:      logger.Named("executor_client"),
+		executorUrl: strings.TrimSuffix(executorUrl, "/"),
+		tappedByUrl: make(map[string]bool),
+		requestChan: make(chan string),
+		httpClient: &http.Client{
+			Transport: &ochttp.Transport{},
+		},
+	}
+	go c.service()
+	return c
+}
+
+func (c *Client) GetServiceForFunction(ctx context.Context, metadata *metav1.ObjectMeta) (string, error) {
+	executorUrl := c.executorUrl + "/v2/getServiceForFunction"
+
+	body, err := json.Marshal(metadata)
+	if err != nil {
+		return "", errors.Wrap(err, "could not marshal request body for getting service for function")
+	}
+
+	resp, err := ctxhttp.Post(ctx, c.httpClient, executorUrl, "application/json", bytes.NewReader(body))
+	if err != nil {
+		return "", errors.Wrap(err, "error posting to getting service for function")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return "", ferror.MakeErrorFromHTTP(resp)
+	}
+
+	svcName, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "error reading response body from getting service for function")
+	}
+
+	return string(svcName), nil
+}
+
+func (c *Client) service() {
+	ticker := time.NewTicker(time.Second * 5)
+	for {
+		select {
+		case serviceUrl := <-c.requestChan:
+			c.tappedByUrl[serviceUrl] = true
+		case <-ticker.C:
+			urls := c.tappedByUrl
+			c.tappedByUrl = make(map[string]bool)
+			if len(urls) > 0 {
+				go func() {
+					for u := range urls {
+						err := c._tapService(u)
+						if err != nil {
+							c.logger.Error("error tapping function service address", zap.Error(err), zap.String("address", u))
+						}
+					}
+					c.logger.Info("tapped services in batch", zap.Int("service_count", len(urls)))
+				}()
+			}
+		}
+	}
+}
+
+func (c *Client) TapService(serviceUrl *url.URL) {
+	c.requestChan <- serviceUrl.String()
+}
+
+func (c *Client) _tapService(serviceUrlStr string) error {
+	executorUrl := c.executorUrl + "/v2/tapService"
+
+	resp, err := http.Post(executorUrl, "application/octet-stream", bytes.NewReader([]byte(serviceUrlStr)))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return ferror.MakeErrorFromHTTP(resp)
+	}
+	return nil
+}

pkg/executor/executor.go

@@ -0,0 +1,251 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package executor
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/dchest/uniuri"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/executor/fscache"
+	"github.com/fission/fission/pkg/executor/newdeploy"
+	"github.com/fission/fission/pkg/executor/poolmgr"
+	"github.com/fission/fission/pkg/executor/reaper"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+	"github.com/fission/fission/pkg/utils"
+)
+
+type (
+	Executor struct {
+		logger *zap.Logger
+
+		gpm *poolmgr.GenericPoolManager
+		ndm *newdeploy.NewDeploy
+
+		fissionClient *crd.FissionClient
+		fsCache       *fscache.FunctionServiceCache
+
+		requestChan chan *createFuncServiceRequest
+		fsCreateWg  map[string]*sync.WaitGroup
+	}
+	createFuncServiceRequest struct {
+		ctx      context.Context
+		funcMeta *metav1.ObjectMeta
+		respChan chan *createFuncServiceResponse
+	}
+
+	createFuncServiceResponse struct {
+		funcSvc *fscache.FuncSvc
+		err     error
+	}
+)
+
+func MakeExecutor(logger *zap.Logger, gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor {
+	executor := &Executor{
+		logger:        logger.Named("executor"),
+		gpm:           gpm,
+		ndm:           ndm,
+		fissionClient: fissionClient,
+		fsCache:       fsCache,
+
+		requestChan: make(chan *createFuncServiceRequest),
+		fsCreateWg:  make(map[string]*sync.WaitGroup),
+	}
+	go executor.serveCreateFuncServices()
+
+	return executor
+}
+
+// All non-cached function service requests go through this goroutine
+// serially. It parallelizes requests for different functions, and
+// ensures that for a given function, only one request causes a pod to
+// get specialized. In other words, it ensures that when there's an
+// ongoing request for a certain function, all other requests wait for
+// that request to complete.
+func (executor *Executor) serveCreateFuncServices() {
+	for {
+		req := <-executor.requestChan
+		m := req.funcMeta
+
+		// Cache miss -- is this first one to request the func?
+		wg, found := executor.fsCreateWg[crd.CacheKey(m)]
+		if !found {
+			// create a waitgroup for other requests for
+			// the same function to wait on
+			wg := &sync.WaitGroup{}
+			wg.Add(1)
+			executor.fsCreateWg[crd.CacheKey(m)] = wg
+
+			// launch a goroutine for each request, to parallelize
+			// the specialization of different functions
+			go func() {
+				fsvc, err := executor.createServiceForFunction(req.ctx, m)
+				req.respChan <- &createFuncServiceResponse{
+					funcSvc: fsvc,
+					err:     err,
+				}
+				delete(executor.fsCreateWg, crd.CacheKey(m))
+				wg.Done()
+			}()
+		} else {
+			// There's an existing request for this function, wait for it to finish
+			go func() {
+				executor.logger.Info("waiting for concurrent request for the same function",
+					zap.Any("function", m))
+				wg.Wait()
+
+				// get the function service from the cache
+				fsvc, err := executor.fsCache.GetByFunction(m)
+
+				// fsCache return error when the entry does not exist/expire.
+				// It normally happened if there are multiple requests are
+				// waiting for the same function and executor failed to cre-
+				// ate service for function.
+				err = errors.Wrapf(err, "error getting service for function",
+					zap.String("function_name", m.Name),
+					zap.String("function_namespace", m.Namespace))
+				req.respChan <- &createFuncServiceResponse{
+					funcSvc: fsvc,
+					err:     err,
+				}
+			}()
+		}
+	}
+}
+
+func (executor *Executor) getFunctionExecutorType(meta *metav1.ObjectMeta) (fv1.ExecutorType, error) {
+	fn, err := executor.fissionClient.Functions(meta.Namespace).Get(meta.Name)
+	if err != nil {
+		return "", err
+	}
+	return fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType, nil
+}
+
+func (executor *Executor) createServiceForFunction(ctx context.Context, meta *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
+	executor.logger.Info("no cached function service found, creating one",
+		zap.String("function_name", meta.Name),
+		zap.String("function_namespace", meta.Namespace))
+
+	executorType, err := executor.getFunctionExecutorType(meta)
+	if err != nil {
+		return nil, err
+	}
+
+	var fsvc *fscache.FuncSvc
+	var fsvcErr error
+
+	switch executorType {
+	case fv1.ExecutorTypeNewdeploy:
+		fsvc, fsvcErr = executor.ndm.GetFuncSvc(ctx, meta)
+	default:
+		fsvc, fsvcErr = executor.gpm.GetFuncSvc(ctx, meta)
+	}
+
+	if fsvcErr != nil {
+		e := "error creating service for function"
+		executor.logger.Error(e,
+			zap.Error(fsvcErr),
+			zap.String("function_name", meta.Name),
+			zap.String("function_namespace", meta.Namespace))
+		fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%s] %s", meta.Name, e))
+	} else if fsvc != nil {
+		_, err = executor.fsCache.Add(*fsvc)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	executor.fsCache.IncreaseColdStarts(meta.Name, string(meta.UID))
+
+	return fsvc, fsvcErr
+}
+
+// isValidAddress invokes isValidService or isValidPod depending on the type of executor
+func (executor *Executor) isValidAddress(fsvc *fscache.FuncSvc) bool {
+	if fsvc.Executor == fscache.NEWDEPLOY {
+		return executor.ndm.IsValid(fsvc)
+	} else {
+		return executor.gpm.IsValid(fsvc)
+	}
+}
+
+func serveMetric(logger *zap.Logger) {
+	// Expose the registered metrics via HTTP.
+	metricAddr := ":8080"
+	http.Handle("/metrics", promhttp.Handler())
+	err := http.ListenAndServe(metricAddr, nil)
+
+	logger.Fatal("done listening on metrics endpoint", zap.Error(err))
+}
+
+// StartExecutor Starts executor and the executor components such as Poolmgr,
+// deploymgr and potential future executor types
+func StartExecutor(logger *zap.Logger, fissionNamespace string, functionNamespace string, envBuilderNamespace string, port int) error {
+	// setup a signal handler for SIGTERM
+	utils.SetupStackTraceHandler()
+
+	fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
+
+	err = fissionClient.WaitForCRDs()
+	if err != nil {
+		return errors.Wrap(err, "error waiting for CRDs")
+	}
+
+	fetcherConfig, err := fetcherConfig.MakeFetcherConfig("/userfunc")
+	if err != nil {
+		return errors.Wrap(err, "Error making fetcher config")
+	}
+
+	restClient := fissionClient.GetCrdClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to get kubernetes client")
+	}
+
+	fsCache := fscache.MakeFunctionServiceCache(logger)
+
+	poolID := strings.ToLower(uniuri.NewLen(8))
+	reaper.CleanupOldExecutorObjects(logger, kubernetesClient, poolID)
+	go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
+
+	gpm := poolmgr.MakeGenericPoolManager(
+		logger,
+		fissionClient, kubernetesClient,
+		functionNamespace, fetcherConfig, poolID)
+
+	ndm := newdeploy.MakeNewDeploy(
+		logger,
+		fissionClient, kubernetesClient, restClient,
+		functionNamespace, fetcherConfig, poolID)
+
+	api := MakeExecutor(logger, gpm, ndm, fissionClient, fsCache)
+
+	go api.Serve(port)
+	go serveMetric(logger)
+
+	return nil
+}

pkg/executor/executor_test.go

@@ -0,0 +1,278 @@
+//
+// This test depends on several env vars:
+//
+//   KUBECONFIG has to point at a kube config with a cluster. The test
+//      will use the default context from that config. Be careful,
+//      don't point this at your production environment. The test is
+//      skipped if KUBECONFIG is undefined.
+//
+//   TEST_SPECIALIZE_URL
+//   TEST_FETCHER_URL
+//      These need to point at <node ip>:30001 and <node ip>:30002,
+//      where <node ip> is the address of any node in the test
+//      cluster.
+//
+//   FETCHER_IMAGE
+//      Optional. Set this to a fetcher image; otherwise uses the
+//      default.
+//
+
+// Here's how I run this on my setup, with minikube:
+// TEST_SPECIALIZE_URL=http://192.168.99.100:30002/specialize TEST_FETCHER_URL=http://192.168.99.100:30001 FETCHER_IMAGE=minikube/fetcher:testing KUBECONFIG=/Users/soam/.kube/config go test -v .
+
+package executor
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	"os"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/executor/client"
+)
+
+func panicIf(err error) {
+	if err != nil {
+		log.Panicf("Error: %v", err)
+	}
+}
+
+// return the number of pods in the given namespace matching the given labels
+func countPods(kubeClient *kubernetes.Clientset, ns string, labelz map[string]string) int {
+	pods, err := kubeClient.CoreV1().Pods(ns).List(metav1.ListOptions{
+		LabelSelector: labels.Set(labelz).AsSelector().String(),
+	})
+	if err != nil {
+		log.Panicf("Failed to list pods: %v", err)
+	}
+	return len(pods.Items)
+}
+
+func createTestNamespace(kubeClient *kubernetes.Clientset, ns string) {
+	_, err := kubeClient.CoreV1().Namespaces().Create(&apiv1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ns,
+		},
+	})
+	if err != nil {
+		log.Panicf("failed to create ns %v: %v", ns, err)
+	}
+	log.Printf("Created namespace %v", ns)
+}
+
+// create a nodeport service
+func createSvc(kubeClient *kubernetes.Clientset, ns string, name string, targetPort int, nodePort int32, labels map[string]string) *apiv1.Service {
+	svc, err := kubeClient.CoreV1().Services(ns).Create(&apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: apiv1.ServiceSpec{
+			Type: apiv1.ServiceTypeNodePort,
+			Ports: []apiv1.ServicePort{
+				{
+					Protocol:   apiv1.ProtocolTCP,
+					Port:       80,
+					TargetPort: intstr.FromInt(targetPort),
+					NodePort:   nodePort,
+				},
+			},
+			Selector: labels,
+		},
+	})
+	if err != nil {
+		log.Panicf("Failed to create svc: %v", err)
+	}
+	return svc
+}
+
+func httpGet(url string) string {
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Panicf("HTTP Get failed: URL %v: %v", url, err)
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Panicf("HTTP Get failed to read body: URL %v: %v", url, err)
+	}
+	return string(body)
+}
+
+func TestExecutor(t *testing.T) {
+	// run in a random namespace so we can have concurrent tests
+	// on a given cluster
+	rand.Seed(time.Now().UTC().UnixNano())
+	testId := rand.Intn(999)
+	fissionNs := fmt.Sprintf("test-%v", testId)
+	functionNs := fmt.Sprintf("test-function-%v", testId)
+
+	// skip test if no cluster available for testing
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if len(kubeconfig) == 0 {
+		t.Skip("Skipping test, no kubernetes cluster")
+		return
+	}
+
+	// connect to k8s
+	// and get CRD client
+	fissionClient, kubeClient, apiExtClient, err := crd.MakeFissionClient()
+	if err != nil {
+		log.Panicf("failed to connect: %v", err)
+	}
+
+	// create the test's namespaces
+	createTestNamespace(kubeClient, fissionNs)
+	defer kubeClient.CoreV1().Namespaces().Delete(fissionNs, nil)
+
+	createTestNamespace(kubeClient, functionNs)
+	defer kubeClient.CoreV1().Namespaces().Delete(functionNs, nil)
+
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	// make sure CRD types exist on cluster
+	err = crd.EnsureFissionCRDs(logger, apiExtClient)
+	if err != nil {
+		log.Panicf("failed to ensure crds: %v", err)
+	}
+
+	err = fissionClient.WaitForCRDs()
+	if err != nil {
+		log.Panicf("failed to wait crds: %v", err)
+	}
+
+	// create an env on the cluster
+	env, err := fissionClient.Environments(fissionNs).Create(&fv1.Environment{
+		Metadata: metav1.ObjectMeta{
+			Name:      "nodejs",
+			Namespace: fissionNs,
+		},
+		Spec: fv1.EnvironmentSpec{
+			Version: 1,
+			Runtime: fv1.Runtime{
+				Image: "fission/node-env",
+			},
+			Builder: fv1.Builder{},
+		},
+	})
+	if err != nil {
+		log.Panicf("failed to create env: %v", err)
+	}
+
+	// create poolmgr
+	port := 9999
+	err = StartExecutor(logger, fissionNs, functionNs, "fission-builder", port)
+	if err != nil {
+		log.Panicf("failed to start poolmgr: %v", err)
+	}
+
+	// connect poolmgr client
+	poolmgrClient := client.MakeClient(logger, fmt.Sprintf("http://localhost:%v", port))
+
+	// Wait for pool to be created (we don't actually need to do
+	// this, since the API should do the right thing in any case).
+	// waitForPool(functionNs, "nodejs")
+	time.Sleep(6 * time.Second)
+
+	envRef := fv1.EnvironmentReference{
+		Namespace: env.Metadata.Namespace,
+		Name:      env.Metadata.Name,
+	}
+
+	deployment := fv1.Archive{
+		Type:    fv1.ArchiveTypeLiteral,
+		Literal: []byte(`module.exports = async function(context) { return { status: 200, body: "Hello, world!\n" }; }`),
+	}
+
+	// create a package
+	p := &fv1.Package{
+		Metadata: metav1.ObjectMeta{
+			Name:      "hello",
+			Namespace: fissionNs,
+		},
+		Spec: fv1.PackageSpec{
+			Environment: envRef,
+			Deployment:  deployment,
+		},
+	}
+	p, err = fissionClient.Packages(fissionNs).Create(p)
+	if err != nil {
+		log.Panicf("failed to create package: %v", err)
+	}
+
+	// create a function
+	f := &fv1.Function{
+		Metadata: metav1.ObjectMeta{
+			Name:      "hello",
+			Namespace: fissionNs,
+		},
+		Spec: fv1.FunctionSpec{
+			Environment: envRef,
+			Package: fv1.FunctionPackageRef{
+				PackageRef: fv1.PackageRef{
+					Namespace:       p.Metadata.Namespace,
+					Name:            p.Metadata.Name,
+					ResourceVersion: p.Metadata.ResourceVersion,
+				},
+			},
+		},
+	}
+	_, err = fissionClient.Functions(fissionNs).Create(f)
+	if err != nil {
+		log.Panicf("failed to create function: %v", err)
+	}
+
+	// create a service to call fetcher and the env container
+	labels := map[string]string{"functionName": f.Metadata.Name}
+	var fetcherPort int32 = 30001
+	fetcherSvc := createSvc(kubeClient, functionNs, fmt.Sprintf("%v-%v", f.Metadata.Name, "fetcher"), 8000, fetcherPort, labels)
+	defer kubeClient.CoreV1().Services(functionNs).Delete(fetcherSvc.ObjectMeta.Name, nil)
+
+	var funcSvcPort int32 = 30002
+	functionSvc := createSvc(kubeClient, functionNs, f.Metadata.Name, 8888, funcSvcPort, labels)
+	defer kubeClient.CoreV1().Services(functionNs).Delete(functionSvc.ObjectMeta.Name, nil)
+
+	// the main test: get a service for a given function
+	t1 := time.Now()
+	svc, err := poolmgrClient.GetServiceForFunction(context.Background(), &f.Metadata)
+	if err != nil {
+		log.Panicf("failed to get func svc: %v", err)
+	}
+	log.Printf("svc for function created at: %v (in %v)", svc, time.Since(t1))
+
+	// ensure that a pod with the label functionName=f.Metadata.Name exists
+	podCount := countPods(kubeClient, functionNs, map[string]string{"functionName": f.Metadata.Name})
+	if podCount != 1 {
+		log.Panicf("expected 1 function pod, found %v", podCount)
+	}
+
+	// call the service to ensure it works
+
+	// wait for a bit
+
+	// tap service to simulate calling it again
+
+	// make sure the same pod is still there
+
+	// wait for idleTimeout to ensure the pod is removed
+
+	// remove env
+
+	// wait for pool to be destroyed
+
+	// that's it
+}

pkg/executor/fscache/functionServiceCache.go

@@ -0,0 +1,294 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fscache
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/cache"
+	"github.com/fission/fission/pkg/crd"
+	ferror "github.com/fission/fission/pkg/error"
+)
+
+type fscRequestType int
+type executorType int
+
+const (
+	TOUCH fscRequestType = iota
+	LISTOLD
+	LOG
+)
+
+const (
+	POOLMGR executorType = iota
+	NEWDEPLOY
+)
+
+type (
+	FuncSvc struct {
+		Name              string                  // Name of object
+		Function          *metav1.ObjectMeta      // function this pod/service is for
+		Environment       *fv1.Environment        // function's environment
+		Address           string                  // Host:Port or IP:Port that the function's service can be reached at.
+		KubernetesObjects []apiv1.ObjectReference // Kubernetes Objects (within the function namespace)
+		Executor          executorType
+
+		Ctime time.Time
+		Atime time.Time
+	}
+
+	FunctionServiceCache struct {
+		logger        *zap.Logger
+		byFunction    *cache.Cache // function-key -> funcSvc  : map[string]*funcSvc
+		byAddress     *cache.Cache // address      -> function : map[string]metav1.ObjectMeta
+		byFunctionUID *cache.Cache // function uid -> function : map[string]metav1.ObjectMeta
+
+		requestChannel chan *fscRequest
+	}
+	fscRequest struct {
+		requestType       fscRequestType
+		address           string
+		kubernetesObjects []apiv1.ObjectReference
+		age               time.Duration
+		responseChannel   chan *fscResponse
+	}
+	fscResponse struct {
+		objects []*FuncSvc
+		deleted bool
+		error
+	}
+)
+
+func IsNotFoundError(err error) bool {
+	if fe, ok := err.(ferror.Error); ok {
+		return fe.Code == ferror.ErrorNotFound
+	}
+	return false
+}
+
+func IsNameExistError(err error) bool {
+	if fe, ok := err.(ferror.Error); ok {
+		return fe.Code == ferror.ErrorNameExists
+	}
+	return false
+}
+
+func MakeFunctionServiceCache(logger *zap.Logger) *FunctionServiceCache {
+	fsc := &FunctionServiceCache{
+		logger:         logger.Named("function_service_cache"),
+		byFunction:     cache.MakeCache(0, 0),
+		byAddress:      cache.MakeCache(0, 0),
+		byFunctionUID:  cache.MakeCache(0, 0),
+		requestChannel: make(chan *fscRequest),
+	}
+	go fsc.service()
+	return fsc
+}
+
+func (fsc *FunctionServiceCache) service() {
+	for {
+		req := <-fsc.requestChannel
+		resp := &fscResponse{}
+		switch req.requestType {
+		case TOUCH:
+			// update atime for this function svc
+			resp.error = fsc._touchByAddress(req.address)
+		case LISTOLD:
+			// get svcs idle for > req.age
+			fscs := fsc.byFunction.Copy()
+			funcObjects := make([]*FuncSvc, 0)
+			for _, funcSvc := range fscs {
+				fsvc := funcSvc.(*FuncSvc)
+				if time.Since(fsvc.Atime) > req.age {
+					funcObjects = append(funcObjects, fsvc)
+				}
+			}
+			resp.objects = funcObjects
+		case LOG:
+			fsc.logger.Info("dumping function service cache")
+			funcCopy := fsc.byFunction.Copy()
+			info := []string{}
+			for key, fsvcI := range funcCopy {
+				fsvc := fsvcI.(*FuncSvc)
+				for _, kubeObj := range fsvc.KubernetesObjects {
+					info = append(info, fmt.Sprintf("%v\t%v\t%v", key, kubeObj.Kind, kubeObj.Name))
+				}
+			}
+			fsc.logger.Info("function service cache", zap.Int("item_count", len(funcCopy)), zap.Strings("cache", info))
+		}
+		req.responseChannel <- resp
+	}
+}
+
+func (fsc *FunctionServiceCache) GetByFunction(m *metav1.ObjectMeta) (*FuncSvc, error) {
+	key := crd.CacheKey(m)
+
+	fsvcI, err := fsc.byFunction.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// update atime
+	fsvc := fsvcI.(*FuncSvc)
+	fsvc.Atime = time.Now()
+
+	fsvcCopy := *fsvc
+	return &fsvcCopy, nil
+}
+
+func (fsc *FunctionServiceCache) GetByFunctionUID(uid types.UID) (*FuncSvc, error) {
+	mI, err := fsc.byFunctionUID.Get(uid)
+	if err != nil {
+		return nil, err
+	}
+
+	m := mI.(metav1.ObjectMeta)
+
+	fsvcI, err := fsc.byFunction.Get(crd.CacheKey(&m))
+	if err != nil {
+		return nil, err
+	}
+
+	// update atime
+	fsvc := fsvcI.(*FuncSvc)
+	fsvc.Atime = time.Now()
+
+	fsvcCopy := *fsvc
+	return &fsvcCopy, nil
+}
+
+func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {
+	err, existing := fsc.byFunction.Set(crd.CacheKey(fsvc.Function), &fsvc)
+	if err != nil {
+		if IsNameExistError(err) {
+			f := existing.(*FuncSvc)
+			err2 := fsc.TouchByAddress(f.Address)
+			if err2 != nil {
+				return nil, err2
+			}
+			fCopy := *f
+			return &fCopy, nil
+		}
+		return nil, err
+	}
+	now := time.Now()
+	fsvc.Ctime = now
+	fsvc.Atime = now
+
+	// Add to byAddress cache. Ignore NameExists errors
+	// because of multiple-specialization. See issue #331.
+	err, _ = fsc.byAddress.Set(fsvc.Address, *fsvc.Function)
+	if err != nil {
+		if IsNameExistError(err) {
+			err = nil
+		} else {
+			err = errors.Wrap(err, "error caching fsvc")
+		}
+		return nil, err
+	}
+
+	// Add to byFunctionUID cache. Ignore NameExists errors
+	// because of multiple-specialization. See issue #331.
+	err, _ = fsc.byFunctionUID.Set(fsvc.Function.UID, *fsvc.Function)
+	if err != nil {
+		if IsNameExistError(err) {
+			err = nil
+		} else {
+			err = errors.Wrap(err, "error caching fsvc by function uid")
+		}
+		return nil, err
+	}
+
+	fsc.setFuncAlive(fsvc.Function.Name, string(fsvc.Function.UID), true)
+	return nil, nil
+}
+
+func (fsc *FunctionServiceCache) TouchByAddress(address string) error {
+	responseChannel := make(chan *fscResponse)
+	fsc.requestChannel <- &fscRequest{
+		requestType:     TOUCH,
+		address:         address,
+		responseChannel: responseChannel,
+	}
+	resp := <-responseChannel
+	return resp.error
+}
+
+func (fsc *FunctionServiceCache) _touchByAddress(address string) error {
+	mI, err := fsc.byAddress.Get(address)
+	if err != nil {
+		return err
+	}
+	m := mI.(metav1.ObjectMeta)
+	fsvcI, err := fsc.byFunction.Get(crd.CacheKey(&m))
+	if err != nil {
+		return err
+	}
+	fsvc := fsvcI.(*FuncSvc)
+	fsvc.Atime = time.Now()
+	return nil
+}
+
+func (fsc *FunctionServiceCache) DeleteEntry(fsvc *FuncSvc) {
+	fsc.byFunction.Delete(crd.CacheKey(fsvc.Function))
+	fsc.byAddress.Delete(fsvc.Address)
+	fsc.byFunctionUID.Delete(fsvc.Function.UID)
+
+	fsc.observeFuncRunningTime(fsvc.Function.Name, string(fsvc.Function.UID), fsvc.Atime.Sub(fsvc.Ctime).Seconds())
+	fsc.observeFuncAliveTime(fsvc.Function.Name, string(fsvc.Function.UID), time.Now().Sub(fsvc.Ctime).Seconds())
+	fsc.setFuncAlive(fsvc.Function.Name, string(fsvc.Function.UID), false)
+}
+
+func (fsc *FunctionServiceCache) DeleteOld(fsvc *FuncSvc, minAge time.Duration) (bool, error) {
+	if time.Since(fsvc.Atime) < minAge {
+		return false, nil
+	}
+
+	fsc.DeleteEntry(fsvc)
+
+	return true, nil
+}
+
+func (fsc *FunctionServiceCache) ListOld(age time.Duration) ([]*FuncSvc, error) {
+	responseChannel := make(chan *fscResponse)
+	fsc.requestChannel <- &fscRequest{
+		requestType:     LISTOLD,
+		age:             age,
+		responseChannel: responseChannel,
+	}
+	resp := <-responseChannel
+	return resp.objects, resp.error
+}
+
+func (fsc *FunctionServiceCache) Log() {
+	fsc.logger.Info("--- FunctionService Cache Contents")
+	responseChannel := make(chan *fscResponse)
+	fsc.requestChannel <- &fscRequest{
+		requestType:     LOG,
+		responseChannel: responseChannel,
+	}
+	<-responseChannel
+	fsc.logger.Info("--- FunctionService Cache Contents End")
+}

pkg/executor/fscache/functionServiceCache_test.go

@@ -0,0 +1,121 @@
+package fscache
+
+import (
+	"log"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func panicIf(err error) {
+	if err != nil {
+		log.Panicf("Error: %v", err)
+	}
+}
+
+func TestFunctionServiceCache(t *testing.T) {
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	fsc := MakeFunctionServiceCache(logger)
+	if fsc == nil {
+		log.Panicf("error creating cache")
+	}
+
+	var fsvc *FuncSvc
+	now := time.Now()
+
+	objects := []apiv1.ObjectReference{
+		{
+			Kind:       "pod",
+			Name:       "xxx",
+			APIVersion: "v1",
+			Namespace:  "fission-function",
+		},
+		{
+			Kind:       "pod",
+			Name:       "xxx2",
+			APIVersion: "v1",
+			Namespace:  "fission-function",
+		},
+	}
+
+	fsvc = &FuncSvc{
+		Function: &metav1.ObjectMeta{
+			Name: "foo",
+			UID:  "1212",
+		},
+		Environment: &fv1.Environment{
+			Metadata: metav1.ObjectMeta{
+				Name: "foo-env",
+				UID:  "2323",
+			},
+			Spec: fv1.EnvironmentSpec{
+				Version: 1,
+				Runtime: fv1.Runtime{
+					Image: "fission/foo-env",
+				},
+				Builder: fv1.Builder{},
+			},
+		},
+		Address:           "xxx",
+		KubernetesObjects: objects,
+		Ctime:             now,
+		Atime:             now,
+	}
+	_, err = fsc.Add(*fsvc)
+	if err != nil {
+		fsc.Log()
+		log.Panicf("Failed to add fsvc: %v", err)
+	}
+
+	f, err := fsc.GetByFunction(fsvc.Function)
+	if err != nil {
+		fsc.Log()
+		log.Panicf("Failed to get fsvc: %v", err)
+	}
+	f, err = fsc.GetByFunctionUID(fsvc.Function.UID)
+	if err != nil {
+		fsc.Log()
+		log.Panicf("Failed to get fsvc by function uid: %v", err)
+	}
+	fsvc.Atime = f.Atime
+	fsvc.Ctime = f.Ctime
+	if f.Address != fsvc.Address {
+		fsc.Log()
+		log.Panicf("Incorrect fsvc \n(expected: %#v)\n (found: %#v)", fsvc, f)
+	}
+
+	err = fsc.TouchByAddress(fsvc.Address)
+	if err != nil {
+		fsc.Log()
+		log.Panicf("Failed to touch fsvc: %v", err)
+	}
+
+	deleted, err := fsc.DeleteOld(fsvc, 0)
+	if err != nil {
+		fsc.Log()
+		log.Panicf("Failed to delete fsvc: %v", err)
+	}
+	if !deleted {
+		fsc.Log()
+		log.Panicf("Did not delete fsvc")
+	}
+
+	_, err = fsc.GetByFunction(fsvc.Function)
+	if err == nil {
+		fsc.Log()
+		log.Panicf("found fsvc while expecting empty cache: %v", err)
+	}
+
+	_, err = fsc.GetByFunctionUID(fsvc.Function.UID)
+	if err == nil {
+		fsc.Log()
+		log.Panicf("found fsvc by function uid while expecting empty cache: %v", err)
+	}
+}

pkg/executor/fscache/metrics.go

@@ -0,0 +1,70 @@
+package fscache
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	metricAddr = ":8080"
+
+	// funcname: the function's name
+	// funcuid: the function's version id
+	coldStarts = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "fission_cold_starts_total",
+			Help: "How many cold starts are made by funcname, funcuid.",
+		},
+		[]string{"funcname", "funcuid"},
+	)
+	funcRunningSummary = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "fission_func_running_seconds_summary",
+			Help:       "The running time (last access - create) in seconds of the function.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"funcname", "funcuid"},
+	)
+	funcAliveSummary = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "fission_func_alive_seconds_summary",
+			Help:       "The alive time in seconds of the function.",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"funcname", "funcuid"},
+	)
+	funcIsAlive = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "fission_func_is_alive",
+			Help: "A binary value indicating is the funcname, funcuid alive",
+		},
+		[]string{"funcname", "funcuid"},
+	)
+)
+
+func init() {
+	// Register the function calls counter with Prometheus's default registry.
+	prometheus.MustRegister(coldStarts)
+	prometheus.MustRegister(funcRunningSummary)
+	prometheus.MustRegister(funcAliveSummary)
+	prometheus.MustRegister(funcIsAlive)
+}
+
+func (fsc *FunctionServiceCache) IncreaseColdStarts(funcname, funcuid string) {
+	coldStarts.WithLabelValues(funcname, funcuid).Inc()
+}
+
+func (fsc *FunctionServiceCache) observeFuncRunningTime(funcname, funcuid string, running float64) {
+	funcRunningSummary.WithLabelValues(funcname, funcuid).Observe(running)
+}
+
+func (fsc *FunctionServiceCache) observeFuncAliveTime(funcname, funcuid string, alive float64) {
+	funcAliveSummary.WithLabelValues(funcname, funcuid).Observe(alive)
+}
+
+func (fsc *FunctionServiceCache) setFuncAlive(funcname, funcuid string, isAlive bool) {
+	count := 0
+	if isAlive {
+		count = 1
+	}
+	funcIsAlive.WithLabelValues(funcname, funcuid).Set(float64(count))
+}

pkg/executor/newdeploy/newdeploy.go

@@ -0,0 +1,447 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package newdeploy
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+	multierror "github.com/hashicorp/go-multierror"
+	"go.uber.org/zap"
+	asv1 "k8s.io/api/autoscaling/v1"
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	k8s_err "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/executor/util"
+)
+
+const (
+	DeploymentKind    = "Deployment"
+	DeploymentVersion = "extensions/v1beta1"
+)
+
+func (deploy *NewDeploy) createOrGetDeployment(fn *fv1.Function, env *fv1.Environment,
+	deployName string, deployLabels map[string]string, deployNamespace string, firstcreate bool) (*v1beta1.Deployment, error) {
+
+	minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
+
+	// If it's not the first time creation and minscale is 0 means that all pods for function were recycled,
+	// in such cases we need set minscale to 1 for router to serve requests.
+	if !firstcreate && minScale <= 0 {
+		minScale = 1
+	}
+
+	waitForDeploy := minScale > 0
+
+	existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Get(deployName, metav1.GetOptions{})
+	if err == nil {
+		if waitForDeploy {
+			err = deploy.scaleDeployment(existingDepl.Namespace, existingDepl.Name, minScale)
+			if err != nil {
+				deploy.logger.Error("error scaling up function deployment", zap.Error(err), zap.String("function", fn.Metadata.Name))
+				return nil, err
+			}
+
+			if existingDepl.Status.AvailableReplicas < minScale {
+				existingDepl, err = deploy.waitForDeploy(existingDepl, minScale)
+			}
+		}
+		return existingDepl, err
+	}
+
+	if err != nil && k8s_err.IsNotFound(err) {
+		err := deploy.setupRBACObjs(deployNamespace, fn)
+		if err != nil {
+			return nil, err
+		}
+
+		deployment, err := deploy.getDeploymentSpec(fn, env, deployName, deployLabels)
+		if err != nil {
+			return nil, err
+		}
+
+		depl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Create(deployment)
+		if err != nil {
+			deploy.logger.Error("error while creating function deployment",
+				zap.Error(err),
+				zap.String("function", fn.Metadata.Name),
+				zap.String("deployment_name", deployName),
+				zap.String("deployment_namespace", deployNamespace))
+			return nil, err
+		}
+
+		if waitForDeploy {
+			depl, err = deploy.waitForDeploy(depl, minScale)
+		}
+
+		return depl, err
+	}
+
+	return nil, err
+
+}
+
+func (deploy *NewDeploy) setupRBACObjs(deployNamespace string, fn *fv1.Function) error {
+	// create fetcher SA in this ns, if not already created
+	err := deploy.fetcherConfig.SetupServiceAccount(deploy.kubernetesClient, deployNamespace, fn.Metadata)
+	if err != nil {
+		deploy.logger.Error("error creating fission fetcher service account for function",
+			zap.Error(err),
+			zap.String("service_account_name", types.FissionFetcherSA),
+			zap.String("service_account_namespace", deployNamespace),
+			zap.String("function_name", fn.Metadata.Name),
+			zap.String("function_namespace", fn.Metadata.Namespace))
+		return err
+	}
+
+	// create a cluster role binding for the fetcher SA, if not already created, granting access to do a get on packages in any ns
+	err = utils.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, types.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, types.PackageGetterCR, types.ClusterRole, types.FissionFetcherSA, deployNamespace)
+	if err != nil {
+		deploy.logger.Error("error creating role binding for function",
+			zap.Error(err),
+			zap.String("role_binding", types.PackageGetterRB),
+			zap.String("function_name", fn.Metadata.Name),
+			zap.String("function_namespace", fn.Metadata.Namespace))
+		return err
+	}
+
+	// create rolebinding in function namespace for fetcherSA.envNamespace to be able to get secrets and configmaps
+	err = utils.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, types.SecretConfigMapGetterRB, fn.Metadata.Namespace, types.SecretConfigMapGetterCR, types.ClusterRole, types.FissionFetcherSA, deployNamespace)
+	if err != nil {
+		deploy.logger.Error("error creating role binding for function",
+			zap.Error(err),
+			zap.String("role_binding", types.SecretConfigMapGetterRB),
+			zap.String("function_name", fn.Metadata.Name),
+			zap.String("function_namespace", fn.Metadata.Namespace))
+		return err
+	}
+
+	deploy.logger.Info("set up all RBAC objects for function",
+		zap.String("function_name", fn.Metadata.Name),
+		zap.String("function_namespace", fn.Metadata.Namespace))
+	return nil
+}
+
+func (deploy *NewDeploy) getDeployment(ns, name string) (*v1beta1.Deployment, error) {
+	return deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Get(name, metav1.GetOptions{})
+}
+
+func (deploy *NewDeploy) updateDeployment(deployment *v1beta1.Deployment, ns string) error {
+	_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Update(deployment)
+	return err
+}
+
+func (deploy *NewDeploy) deleteDeployment(ns string, name string) error {
+	// DeletePropagationBackground deletes the object immediately and dependent are deleted later
+	// DeletePropagationForeground not advisable; it markes for deleteion and API can still serve those objects
+	deletePropagation := metav1.DeletePropagationBackground
+	err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Delete(name, &metav1.DeleteOptions{
+		PropagationPolicy: &deletePropagation,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (deploy *NewDeploy) getDeploymentSpec(fn *fv1.Function, env *fv1.Environment,
+	deployName string, deployLabels map[string]string) (*v1beta1.Deployment, error) {
+
+	replicas := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
+
+	gracePeriodSeconds := int64(6 * 60)
+	if env.Spec.TerminationGracePeriod > 0 {
+		gracePeriodSeconds = env.Spec.TerminationGracePeriod
+	}
+
+	podAnnotations := env.Metadata.Annotations
+	if podAnnotations == nil {
+		podAnnotations = make(map[string]string)
+	}
+	if deploy.useIstio && env.Spec.AllowAccessToExternalNetwork {
+		podAnnotations["sidecar.istio.io/inject"] = "false"
+	}
+	resources := deploy.getResources(env, fn)
+
+	deployment := &v1beta1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   deployName,
+			Labels: deployLabels,
+		},
+		Spec: v1beta1.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: deployLabels,
+			},
+			Template: apiv1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      deployLabels,
+					Annotations: podAnnotations,
+				},
+				Spec: apiv1.PodSpec{
+					Containers: []apiv1.Container{
+						util.MergeContainerSpecs(&apiv1.Container{
+							Name:                   fn.Metadata.Name,
+							Image:                  env.Spec.Runtime.Image,
+							ImagePullPolicy:        deploy.runtimeImagePullPolicy,
+							TerminationMessagePath: "/dev/termination-log",
+							Lifecycle: &apiv1.Lifecycle{
+								PreStop: &apiv1.Handler{
+									Exec: &apiv1.ExecAction{
+										Command: []string{
+											"/bin/sleep",
+											fmt.Sprintf("%v", gracePeriodSeconds),
+										},
+									},
+								},
+							},
+							Resources: resources,
+						}, env.Spec.Runtime.Container),
+					},
+					ServiceAccountName:            "fission-fetcher",
+					TerminationGracePeriodSeconds: &gracePeriodSeconds,
+				},
+			},
+		},
+	}
+
+	// Order of merging is important here - first fetcher, then containers and lastly pod spec
+	err := deploy.fetcherConfig.AddSpecializingFetcherToPodSpec(
+		&deployment.Spec.Template.Spec,
+		fn.Metadata.Name,
+		fn,
+		env,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	if env.Spec.Runtime.PodSpec != nil {
+		err := util.MergePodSpec(&deployment.Spec.Template.Spec, env.Spec.Runtime.PodSpec)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return deployment, nil
+}
+
+// getResources overrides only the resources which are overridden at function level otherwise
+// default to resources specified at environment level
+func (deploy *NewDeploy) getResources(env *fv1.Environment, fn *fv1.Function) apiv1.ResourceRequirements {
+	resources := env.Spec.Resources
+	if resources.Requests == nil {
+		resources.Requests = make(map[apiv1.ResourceName]resource.Quantity)
+	}
+	if resources.Limits == nil {
+		resources.Limits = make(map[apiv1.ResourceName]resource.Quantity)
+	}
+	// Only override the once specified at function, rest default to values from env.
+	val, ok := fn.Spec.Resources.Requests[apiv1.ResourceCPU]
+	if ok && !val.IsZero() {
+		resources.Requests[apiv1.ResourceCPU] = fn.Spec.Resources.Requests[apiv1.ResourceCPU]
+	}
+
+	val, ok = fn.Spec.Resources.Requests[apiv1.ResourceMemory]
+	if ok && !val.IsZero() {
+		resources.Requests[apiv1.ResourceMemory] = fn.Spec.Resources.Requests[apiv1.ResourceMemory]
+	}
+
+	val, ok = fn.Spec.Resources.Limits[apiv1.ResourceCPU]
+	if ok && !val.IsZero() {
+		resources.Limits[apiv1.ResourceCPU] = fn.Spec.Resources.Limits[apiv1.ResourceCPU]
+	}
+
+	val, ok = fn.Spec.Resources.Limits[apiv1.ResourceMemory]
+	if ok && !val.IsZero() {
+		resources.Limits[apiv1.ResourceMemory] = fn.Spec.Resources.Limits[apiv1.ResourceMemory]
+	}
+
+	return resources
+}
+
+func (deploy *NewDeploy) createOrGetHpa(hpaName string, execStrategy *fv1.ExecutionStrategy, depl *v1beta1.Deployment) (*asv1.HorizontalPodAutoscaler, error) {
+
+	minRepl := int32(execStrategy.MinScale)
+	if minRepl == 0 {
+		minRepl = 1
+	}
+	maxRepl := int32(execStrategy.MaxScale)
+	if maxRepl == 0 {
+		maxRepl = minRepl
+	}
+	targetCPU := int32(execStrategy.TargetCPUPercent)
+
+	existingHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Get(hpaName, metav1.GetOptions{})
+	if err == nil {
+		return existingHpa, err
+	}
+
+	if depl == nil {
+		return nil, errors.New("failed to create HPA, found empty deployment")
+	}
+
+	if err != nil && k8s_err.IsNotFound(err) {
+		hpa := asv1.HorizontalPodAutoscaler{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:   hpaName,
+				Labels: depl.Labels,
+			},
+			Spec: asv1.HorizontalPodAutoscalerSpec{
+				ScaleTargetRef: asv1.CrossVersionObjectReference{
+					Kind:       DeploymentKind,
+					Name:       depl.ObjectMeta.Name,
+					APIVersion: DeploymentVersion,
+				},
+				MinReplicas:                    &minRepl,
+				MaxReplicas:                    maxRepl,
+				TargetCPUUtilizationPercentage: &targetCPU,
+			},
+		}
+
+		cHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Create(&hpa)
+		if err != nil {
+			return nil, err
+		}
+		return cHpa, nil
+	}
+
+	return nil, err
+
+}
+
+func (deploy *NewDeploy) getHpa(ns, name string) (*asv1.HorizontalPodAutoscaler, error) {
+	return deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(ns).Get(name, metav1.GetOptions{})
+}
+
+func (deploy *NewDeploy) updateHpa(hpa *asv1.HorizontalPodAutoscaler) error {
+	_, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Update(hpa)
+	return err
+}
+
+func (deploy *NewDeploy) deleteHpa(ns string, name string) error {
+	err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(ns).Delete(name, &metav1.DeleteOptions{})
+	return err
+}
+
+func (deploy *NewDeploy) createOrGetSvc(deployLabels map[string]string, svcName string, svcNamespace string) (*apiv1.Service, error) {
+
+	existingSvc, err := deploy.kubernetesClient.CoreV1().Services(svcNamespace).Get(svcName, metav1.GetOptions{})
+	if err == nil {
+		return existingSvc, err
+	}
+
+	if err != nil && k8s_err.IsNotFound(err) {
+		service := &apiv1.Service{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:   svcName,
+				Labels: deployLabels,
+			},
+			Spec: apiv1.ServiceSpec{
+				Ports: []apiv1.ServicePort{
+					{
+						Name:       "runtime-env-port",
+						Port:       int32(80),
+						TargetPort: intstr.FromInt(8888),
+					},
+					{
+						Name:       "fetcher-port",
+						Port:       int32(8000),
+						TargetPort: intstr.FromInt(8000),
+					},
+				},
+				Selector: deployLabels,
+				Type:     apiv1.ServiceTypeClusterIP,
+			},
+		}
+
+		svc, err := deploy.kubernetesClient.CoreV1().Services(svcNamespace).Create(service)
+		if err != nil {
+			return nil, err
+		}
+
+		return svc, nil
+	}
+
+	return nil, err
+}
+
+func (deploy *NewDeploy) deleteSvc(ns string, name string) error {
+	err := deploy.kubernetesClient.CoreV1().Services(ns).Delete(name, &metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (deploy *NewDeploy) waitForDeploy(depl *v1beta1.Deployment, replicas int32) (*v1beta1.Deployment, error) {
+	for i := 0; i < 120; i++ {
+		latestDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(depl.ObjectMeta.Namespace).Get(depl.Name, metav1.GetOptions{})
+		if err != nil {
+			return nil, err
+		}
+		//TODO check for imagePullerror
+		// use AvailableReplicas here is better than ReadyReplicas
+		// since the pods may not be able to serve network traffic yet.
+		if latestDepl.Status.AvailableReplicas >= replicas {
+			return latestDepl, err
+		}
+		time.Sleep(time.Second)
+	}
+	return nil, errors.New("failed to create deployment within timeout window")
+}
+
+// cleanupNewdeploy cleans all kubernetes objects related to function
+func (deploy *NewDeploy) cleanupNewdeploy(ns string, name string) error {
+	var multierr *multierror.Error
+
+	err := deploy.deleteSvc(ns, name)
+	if err != nil {
+		deploy.logger.Error("error deleting service for newdeploy function",
+			zap.Error(err),
+			zap.String("function_name", name),
+			zap.String("function_namespace", ns))
+		multierror.Append(multierr, err)
+	}
+
+	err = deploy.deleteHpa(ns, name)
+	if err != nil {
+		deploy.logger.Error("error deleting service for newdeploy function",
+			zap.Error(err),
+			zap.String("function_name", name),
+			zap.String("function_namespace", ns))
+		multierror.Append(multierr, err)
+	}
+
+	err = deploy.deleteDeployment(ns, name)
+	if err != nil {
+		deploy.logger.Error("error deleting deployment for newdeploy function",
+			zap.Error(err),
+			zap.String("function_name", name),
+			zap.String("function_namespace", ns))
+		multierror.Append(multierr, err)
+	}
+	return multierr.ErrorOrNil()
+}

pkg/executor/newdeploy/newdeploymgr.go

@@ -0,0 +1,722 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package newdeploy
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/dchest/uniuri"
+	"github.com/fission/fission/pkg/throttler"
+	"github.com/fission/fission/pkg/utils"
+	multierror "github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	k8sErrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	k8sTypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/executor/fscache"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+	"github.com/fission/fission/pkg/types"
+)
+
+type (
+	NewDeploy struct {
+		logger *zap.Logger
+
+		kubernetesClient *kubernetes.Clientset
+		fissionClient    *crd.FissionClient
+		crdClient        *rest.RESTClient
+		instanceID       string
+		fetcherConfig    *fetcherConfig.Config
+
+		runtimeImagePullPolicy apiv1.PullPolicy
+		namespace              string
+		useIstio               bool
+		collectorEndpoint      string
+
+		fsCache *fscache.FunctionServiceCache // cache funcSvc's by function, address and pod name
+
+		throttler      *throttler.Throttler
+		funcStore      k8sCache.Store
+		funcController k8sCache.Controller
+
+		envStore      k8sCache.Store
+		envController k8sCache.Controller
+
+		idlePodReapTime time.Duration
+	}
+)
+
+func MakeNewDeploy(
+	logger *zap.Logger,
+	fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset,
+	crdClient *rest.RESTClient,
+	namespace string,
+	fetcherConfig *fetcherConfig.Config,
+	instanceID string,
+) *NewDeploy {
+
+	logger.Info("creating NewDeploy ExecutorType")
+
+	enableIstio := false
+	if len(os.Getenv("ENABLE_ISTIO")) > 0 {
+		istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
+		if err != nil {
+			logger.Error("failed to parse 'ENABLE_ISTIO', set to false", zap.Error(err))
+		}
+		enableIstio = istio
+	}
+
+	nd := &NewDeploy{
+		logger: logger.Named("new_deploy"),
+
+		fissionClient:    fissionClient,
+		kubernetesClient: kubernetesClient,
+		crdClient:        crdClient,
+		instanceID:       instanceID,
+
+		namespace: namespace,
+		fsCache:   fscache.MakeFunctionServiceCache(logger),
+		throttler: throttler.MakeThrottler(1 * time.Minute),
+
+		fetcherConfig:          fetcherConfig,
+		runtimeImagePullPolicy: utils.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY")),
+		useIstio:               enableIstio,
+
+		idlePodReapTime: 2 * time.Minute,
+	}
+
+	if nd.crdClient != nil {
+		fnStore, fnController := nd.initFuncController()
+		nd.funcStore = fnStore
+		nd.funcController = fnController
+
+		envStore, envController := nd.initEnvController()
+		nd.envStore = envStore
+		nd.envController = envController
+	}
+
+	return nd
+}
+
+func (deploy *NewDeploy) Run(ctx context.Context) {
+	//go deploy.service()
+	go deploy.funcController.Run(ctx.Done())
+	go deploy.envController.Run(ctx.Done())
+	go deploy.idleObjectReaper()
+}
+
+func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controller) {
+	resyncPeriod := 30 * time.Second
+	listWatch := k8sCache.NewListWatchFromClient(deploy.crdClient, "functions", metav1.NamespaceAll, fields.Everything())
+	store, controller := k8sCache.NewInformer(listWatch, &fv1.Function{}, resyncPeriod, k8sCache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			fn := obj.(*fv1.Function)
+			_, err := deploy.createFunction(fn, true)
+			if err != nil {
+				deploy.logger.Error("error eager creating function",
+					zap.Error(err),
+					zap.Any("function", fn))
+			}
+		},
+		DeleteFunc: func(obj interface{}) {
+			fn := obj.(*fv1.Function)
+			err := deploy.deleteFunction(fn)
+			if err != nil {
+				deploy.logger.Error("error deleting function",
+					zap.Error(err),
+					zap.Any("function", fn))
+			}
+		},
+		UpdateFunc: func(oldObj interface{}, newObj interface{}) {
+			oldFn := oldObj.(*fv1.Function)
+			newFn := newObj.(*fv1.Function)
+			err := deploy.updateFunction(oldFn, newFn)
+			if err != nil {
+				deploy.logger.Error("error updating function",
+					zap.Error(err),
+					zap.Any("old_function", oldFn),
+					zap.Any("new_function", newFn))
+			}
+		},
+	})
+	return store, controller
+}
+
+func (deploy *NewDeploy) initEnvController() (k8sCache.Store, k8sCache.Controller) {
+	resyncPeriod := 30 * time.Second
+	listWatch := k8sCache.NewListWatchFromClient(deploy.crdClient, "environments", metav1.NamespaceAll, fields.Everything())
+	store, controller := k8sCache.NewInformer(listWatch, &fv1.Environment{}, resyncPeriod, k8sCache.ResourceEventHandlerFuncs{
+		AddFunc:    func(obj interface{}) {},
+		DeleteFunc: func(obj interface{}) {},
+		UpdateFunc: func(oldObj interface{}, newObj interface{}) {
+			newEnv := newObj.(*fv1.Environment)
+			oldEnv := oldObj.(*fv1.Environment)
+			// Currently only an image update in environment calls for function's deployment recreation. In future there might be more attributes which would want to do it
+			if oldEnv.Spec.Runtime.Image != newEnv.Spec.Runtime.Image {
+				deploy.logger.Info("Updating all function of the environment that changed, old env:", zap.Any("environment", oldEnv))
+				funcs := deploy.getEnvFunctions(&newEnv.Metadata)
+				for _, f := range funcs {
+					function, err := deploy.fissionClient.Functions(f.Metadata.Namespace).Get(f.Metadata.Name)
+					if err != nil {
+						deploy.logger.Error("Error getting function", zap.Error(err), zap.Any("function", function))
+					}
+					err = deploy.updateFuncDeployment(function, newEnv)
+					if err != nil {
+						deploy.logger.Error("Error updating function", zap.Error(err), zap.Any("function", function))
+					}
+				}
+			}
+		},
+	})
+	return store, controller
+}
+
+func (deploy *NewDeploy) getEnvFunctions(m *metav1.ObjectMeta) []fv1.Function {
+	funcList, err := deploy.fissionClient.Functions(m.Namespace).List(metav1.ListOptions{})
+	if err != nil {
+		deploy.logger.Error("Error getting functions for env", zap.Error(err), zap.Any("environment", m))
+	}
+	relatedFunctions := make([]fv1.Function, 0)
+	for _, f := range funcList.Items {
+		if (f.Spec.Environment.Name == m.Name) && (f.Spec.Environment.Namespace == m.Namespace) {
+			relatedFunctions = append(relatedFunctions, f)
+		}
+	}
+	return relatedFunctions
+}
+
+func (deploy *NewDeploy) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
+	fn, err := deploy.fissionClient.Functions(metadata.Namespace).Get(metadata.Name)
+	if err != nil {
+		return nil, err
+	}
+	return deploy.createFunction(fn, false)
+}
+
+func (deploy *NewDeploy) createFunction(fn *fv1.Function, firstcreate bool) (*fscache.FuncSvc, error) {
+	if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
+		return nil, nil
+	}
+
+	fsvcObj, err := deploy.throttler.RunOnce(string(fn.Metadata.UID), func(ableToCreate bool) (interface{}, error) {
+		if ableToCreate {
+			return deploy.fnCreate(fn, firstcreate)
+		}
+		return deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
+	})
+
+	fsvc, ok := fsvcObj.(*fscache.FuncSvc)
+	if !ok {
+		deploy.logger.Panic("receive unknown object while creating function - expected pointer of function service object")
+	}
+
+	return fsvc, err
+}
+
+func (deploy *NewDeploy) deleteFunction(fn *fv1.Function) error {
+	if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
+		return nil
+	}
+	err := deploy.fnDelete(fn)
+	if err != nil {
+		err = errors.Wrapf(err, "error deleting kubernetes objects of function %v", fn.Metadata)
+	}
+	return err
+}
+
+func (deploy *NewDeploy) fnCreate(fn *fv1.Function, firstcreate bool) (*fscache.FuncSvc, error) {
+	env, err := deploy.fissionClient.
+		Environments(fn.Spec.Environment.Namespace).
+		Get(fn.Spec.Environment.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	objName := deploy.getObjName(fn)
+	if !firstcreate {
+		// retrieve back the previous obj name for later use.
+		fsvc, err := deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
+		if err == nil {
+			objName = fsvc.Name
+		}
+	}
+	deployLabels := deploy.getDeployLabels(fn, env)
+
+	// to support backward compatibility, if the function was created in default ns, we fall back to creating the
+	// deployment of the function in fission-function ns
+	ns := deploy.namespace
+	if fn.Metadata.Namespace != metav1.NamespaceDefault {
+		ns = fn.Metadata.Namespace
+	}
+
+	// Envoy(istio-proxy) returns 404 directly before istio pilot
+	// propagates latest Envoy-specific configuration.
+	// Since newdeploy waits for pods of deployment to be ready,
+	// change the order of kubeObject creation (create service first,
+	// then deployment) to take advantage of waiting time.
+	svc, err := deploy.createOrGetSvc(deployLabels, objName, ns)
+	if err != nil {
+		deploy.logger.Error("error creating service", zap.Error(err), zap.String("service", objName))
+		go deploy.cleanupNewdeploy(ns, objName)
+		return nil, errors.Wrapf(err, "error creating service %v", objName)
+	}
+	svcAddress := fmt.Sprintf("%v.%v", svc.Name, svc.Namespace)
+	depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels, ns, firstcreate)
+	if err != nil {
+		deploy.logger.Error("error creating deployment", zap.Error(err), zap.String("deployment", objName))
+		go deploy.cleanupNewdeploy(ns, objName)
+		return nil, errors.Wrapf(err, "error creating deployment %v", objName)
+	}
+
+	hpa, err := deploy.createOrGetHpa(objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl)
+	if err != nil {
+		deploy.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName))
+		go deploy.cleanupNewdeploy(ns, objName)
+		return nil, errors.Wrapf(err, "error creating the HPA %v", objName)
+	}
+
+	kubeObjRefs := []apiv1.ObjectReference{
+		{
+			//obj.TypeMeta.Kind does not work hence this, needs investigation and a fix
+			Kind:            "deployment",
+			Name:            depl.ObjectMeta.Name,
+			APIVersion:      depl.TypeMeta.APIVersion,
+			Namespace:       depl.ObjectMeta.Namespace,
+			ResourceVersion: depl.ObjectMeta.ResourceVersion,
+			UID:             depl.ObjectMeta.UID,
+		},
+		{
+			Kind:            "service",
+			Name:            svc.ObjectMeta.Name,
+			APIVersion:      svc.TypeMeta.APIVersion,
+			Namespace:       svc.ObjectMeta.Namespace,
+			ResourceVersion: svc.ObjectMeta.ResourceVersion,
+			UID:             svc.ObjectMeta.UID,
+		},
+		{
+			Kind:            "horizontalpodautoscaler",
+			Name:            hpa.ObjectMeta.Name,
+			APIVersion:      hpa.TypeMeta.APIVersion,
+			Namespace:       hpa.ObjectMeta.Namespace,
+			ResourceVersion: hpa.ObjectMeta.ResourceVersion,
+			UID:             hpa.ObjectMeta.UID,
+		},
+	}
+
+	fsvc := &fscache.FuncSvc{
+		Name:              objName,
+		Function:          &fn.Metadata,
+		Environment:       env,
+		Address:           svcAddress,
+		KubernetesObjects: kubeObjRefs,
+		Executor:          fscache.NEWDEPLOY,
+	}
+
+	_, err = deploy.fsCache.Add(*fsvc)
+	if err != nil {
+		deploy.logger.Error("error adding function to cache", zap.Error(err), zap.Any("function", fsvc.Function))
+		return fsvc, err
+	}
+	return fsvc, nil
+}
+
+func (deploy *NewDeploy) updateFunction(oldFn *fv1.Function, newFn *fv1.Function) error {
+
+	if oldFn.Metadata.ResourceVersion == newFn.Metadata.ResourceVersion {
+		return nil
+	}
+
+	// Ignoring updates to functions which are not of NewDeployment type
+	if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy &&
+		oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
+		return nil
+	}
+
+	// Executor type is no longer New Deployment
+	if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy &&
+		oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypeNewdeploy {
+		deploy.logger.Info("function does not use new deployment executor anymore, deleting resources",
+			zap.Any("function", newFn))
+		// IMP - pass the oldFn, as the new/modified function is not in cache
+		return deploy.deleteFunction(oldFn)
+	}
+
+	// Executor type changed to New Deployment from something else
+	if oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy &&
+		newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypeNewdeploy {
+		deploy.logger.Info("function type changed to new deployment, creating resources",
+			zap.Any("old_function", oldFn.Metadata),
+			zap.Any("new_function", newFn.Metadata))
+		_, err := deploy.createFunction(newFn, true)
+		if err != nil {
+			deploy.updateStatus(oldFn, err, "error changing the function's type to newdeploy")
+		}
+		return err
+	}
+
+	deployChanged := false
+
+	if oldFn.Spec.InvokeStrategy != newFn.Spec.InvokeStrategy {
+
+		// to support backward compatibility, if the function was created in default ns, we fall back to creating the
+		// deployment of the function in fission-function ns, so cleaning up resources there
+		ns := deploy.namespace
+		if newFn.Metadata.Namespace != metav1.NamespaceDefault {
+			ns = newFn.Metadata.Namespace
+		}
+
+		fsvc, err := deploy.fsCache.GetByFunctionUID(newFn.Metadata.UID)
+		if err != nil {
+			err = errors.Wrapf(err, "error updating function due to unable to find function service cache: %v", oldFn)
+			return err
+		}
+
+		hpa, err := deploy.getHpa(ns, fsvc.Name)
+		if err != nil {
+			deploy.updateStatus(oldFn, err, "error getting HPA while updating function")
+			return err
+		}
+
+		hpaChanged := false
+
+		if newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale {
+			replicas := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
+			hpa.Spec.MinReplicas = &replicas
+			hpaChanged = true
+		}
+
+		if newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale {
+			hpa.Spec.MaxReplicas = int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale)
+			hpaChanged = true
+		}
+
+		if newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent != oldFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent {
+			targetCpupercent := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent)
+			hpa.Spec.TargetCPUUtilizationPercentage = &targetCpupercent
+			hpaChanged = true
+		}
+
+		if hpaChanged {
+			err := deploy.updateHpa(hpa)
+			if err != nil {
+				deploy.updateStatus(oldFn, err, "error updating HPA while updating function")
+				return err
+			}
+		}
+	}
+
+	if oldFn.Spec.Environment != newFn.Spec.Environment ||
+		oldFn.Spec.Package.PackageRef != newFn.Spec.Package.PackageRef ||
+		oldFn.Spec.Package.FunctionName != newFn.Spec.Package.FunctionName {
+		deployChanged = true
+	}
+
+	// If length of slice has changed then no need to check individual elements
+	if len(oldFn.Spec.Secrets) != len(newFn.Spec.Secrets) {
+		deployChanged = true
+	} else {
+		for i, newSecret := range newFn.Spec.Secrets {
+			if newSecret != oldFn.Spec.Secrets[i] {
+				deployChanged = true
+				break
+			}
+		}
+	}
+	if len(oldFn.Spec.ConfigMaps) != len(newFn.Spec.ConfigMaps) {
+		deployChanged = true
+	} else {
+		for i, newConfig := range newFn.Spec.ConfigMaps {
+			if newConfig != oldFn.Spec.ConfigMaps[i] {
+				deployChanged = true
+				break
+			}
+		}
+	}
+
+	if deployChanged == true {
+		env, err := deploy.fissionClient.Environments(newFn.Spec.Environment.Namespace).
+			Get(newFn.Spec.Environment.Name)
+		if err != nil {
+			deploy.updateStatus(oldFn, err, "failed to get environment while updating function")
+			return err
+		}
+		return deploy.updateFuncDeployment(newFn, env)
+	}
+
+	return nil
+}
+
+func (deploy *NewDeploy) updateFuncDeployment(fn *fv1.Function, env *fv1.Environment) error {
+
+	fsvc, err := deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
+	if err != nil {
+		err = errors.Wrapf(err, "error updating function due to unable to find function service cache: %v", fn)
+		return err
+	}
+	fnObjName := fsvc.Name
+
+	deployLabels := deploy.getDeployLabels(fn, env)
+	deploy.logger.Info("updating deployment due to function/environment update", zap.String("deployment", fnObjName), zap.Any("function", fn.Metadata.Name))
+
+	newDeployment, err := deploy.getDeploymentSpec(fn, env, fnObjName, deployLabels)
+	if err != nil {
+		deploy.updateStatus(fn, err, "failed to get new deployment spec while updating function")
+		return err
+	}
+
+	// to support backward compatibility, if the function was created in default ns, we fall back to creating the
+	// deployment of the function in fission-function ns
+	ns := deploy.namespace
+	if fn.Metadata.Namespace != metav1.NamespaceDefault {
+		ns = fn.Metadata.Namespace
+	}
+
+	err = deploy.updateDeployment(newDeployment, ns)
+	if err != nil {
+		deploy.updateStatus(fn, err, "failed to update deployment while updating function")
+		return err
+	}
+
+	return nil
+}
+
+func (deploy *NewDeploy) fnDelete(fn *fv1.Function) error {
+	var multierr *multierror.Error
+
+	// GetByFunction uses resource version as part of cache key, however,
+	// the resource version in function metadata will be changed when a function
+	// is deleted and cause newdeploy backend fails to delete the entry.
+	// Use GetByFunctionUID instead of GetByFunction here to find correct
+	// fsvc entry.
+	fsvc, err := deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("fsvc not found in cache: %v", fn.Metadata))
+		return err
+	}
+
+	objName := fsvc.Name
+
+	_, err = deploy.fsCache.DeleteOld(fsvc, time.Second*0)
+	if err != nil {
+		multierr = multierror.Append(multierr,
+			errors.Wrap(err, fmt.Sprintf("error deleting the function from cache")))
+	}
+
+	// to support backward compatibility, if the function was created in default ns, we fall back to creating the
+	// deployment of the function in fission-function ns, so cleaning up resources there
+	ns := deploy.namespace
+	if fn.Metadata.Namespace != metav1.NamespaceDefault {
+		ns = fn.Metadata.Namespace
+	}
+
+	err = deploy.cleanupNewdeploy(ns, objName)
+	multierr = multierror.Append(multierr, err)
+
+	return multierr.ErrorOrNil()
+}
+
+// getObjName returns a unique name for kubernetes objects of function
+func (deploy *NewDeploy) getObjName(fn *fv1.Function) string {
+	return strings.ToLower(fmt.Sprintf("newdeploy-%v-%v-%v", fn.Metadata.Name, fn.Metadata.Namespace, uniuri.NewLen(8)))
+}
+
+func (deploy *NewDeploy) getDeployLabels(fn *fv1.Function, env *fv1.Environment) map[string]string {
+	return map[string]string{
+		types.EXECUTOR_INSTANCEID_LABEL: deploy.instanceID,
+		types.EXECUTOR_TYPE:             fv1.ExecutorTypeNewdeploy,
+		types.ENVIRONMENT_NAME:          env.Metadata.Name,
+		types.ENVIRONMENT_NAMESPACE:     env.Metadata.Namespace,
+		types.ENVIRONMENT_UID:           string(env.Metadata.UID),
+		types.FUNCTION_NAME:             fn.Metadata.Name,
+		types.FUNCTION_NAMESPACE:        fn.Metadata.Namespace,
+		types.FUNCTION_UID:              string(fn.Metadata.UID),
+	}
+}
+
+// updateKubeObjRefRV update the resource version of kubeObjectRef with
+// given kind and return error if failed to find the reference.
+func (deploy *NewDeploy) updateKubeObjRefRV(fsvc *fscache.FuncSvc, objKind string, rv string) error {
+	kubeObjs := fsvc.KubernetesObjects
+	for i, obj := range kubeObjs {
+		if obj.Kind == objKind {
+			kubeObjs[i].ResourceVersion = rv
+			return nil
+		}
+	}
+	fsvc.KubernetesObjects = kubeObjs
+	return fmt.Errorf("error finding kubernetes object reference with kind: %v", objKind)
+}
+
+// updateStatus is a function which updates status of update.
+// Current implementation only logs messages, in future it will update function status
+func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message string) {
+	deploy.logger.Info("function status update", zap.Error(err), zap.Any("function", fn), zap.String("message", message))
+}
+
+// IsValid does a get on the service address to ensure it's a valid service, then
+// scale deployment to 1 replica if there are no available replicas for function.
+// Return true if no error occurs, return false otherwise.
+func (deploy *NewDeploy) IsValid(fsvc *fscache.FuncSvc) bool {
+	service := strings.Split(fsvc.Address, ".")
+	if len(service) == 0 {
+		return false
+	}
+
+	_, err := deploy.kubernetesClient.CoreV1().Services(service[1]).Get(service[0], metav1.GetOptions{})
+	if err != nil {
+		deploy.logger.Error("error validating function service address", zap.String("function", fsvc.Function.Name), zap.Error(err))
+		return false
+	}
+
+	deployObj := getDeploymentObj(fsvc.KubernetesObjects)
+	if deployObj == nil {
+		deploy.logger.Error("deployment obj for function does not exist", zap.String("function", fsvc.Function.Name))
+		return false
+	}
+
+	currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
+		Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
+	if err != nil {
+		deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+		return false
+	}
+
+	// return directly when available replicas > 0
+	if currentDeploy.Status.AvailableReplicas > 0 {
+		return true
+	}
+
+	return false
+}
+
+// idleObjectReaper reaps objects after certain idle time
+func (deploy *NewDeploy) idleObjectReaper() {
+
+	pollSleep := time.Duration(deploy.idlePodReapTime)
+	for {
+		time.Sleep(pollSleep)
+
+		envs, err := deploy.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
+		if err != nil {
+			deploy.logger.Fatal("failed to get environment list", zap.Error(err))
+		}
+
+		envList := make(map[k8sTypes.UID]struct{})
+		for _, env := range envs.Items {
+			envList[env.Metadata.UID] = struct{}{}
+		}
+
+		funcSvcs, err := deploy.fsCache.ListOld(deploy.idlePodReapTime)
+		if err != nil {
+			deploy.logger.Error("error reaping idle pods", zap.Error(err))
+			continue
+		}
+
+		for _, fsvc := range funcSvcs {
+			if fsvc.Executor != fscache.NEWDEPLOY {
+				continue
+			}
+
+			// For function with the environment that no longer exists, executor
+			// scales down the deployment as usual and prints log to notify user.
+			if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
+				deploy.logger.Error("function environment no longer exists",
+					zap.String("environment", fsvc.Environment.Metadata.Name),
+					zap.String("function", fsvc.Name))
+			}
+
+			fn, err := deploy.fissionClient.Functions(fsvc.Function.Namespace).Get(fsvc.Function.Name)
+			if err != nil {
+				// Newdeploy manager handles the function delete event and clean cache/kubeobjs itself,
+				// so we ignore the not found error for functions with newdeploy executor type here.
+				if k8sErrs.IsNotFound(err) && fsvc.Executor == fscache.NEWDEPLOY {
+					continue
+				}
+				deploy.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name))
+				continue
+			}
+
+			deployObj := getDeploymentObj(fsvc.KubernetesObjects)
+			if deployObj == nil {
+				deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+				continue
+			}
+
+			currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
+				Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
+			if err != nil {
+				deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+				continue
+			}
+
+			minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
+
+			// do nothing if the current replicas is already lower than minScale
+			if *currentDeploy.Spec.Replicas <= minScale {
+				continue
+			}
+
+			err = deploy.scaleDeployment(deployObj.Namespace, deployObj.Name, minScale)
+			if err != nil {
+				deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
+			}
+		}
+	}
+}
+
+func getDeploymentObj(kubeobjs []apiv1.ObjectReference) *apiv1.ObjectReference {
+	for _, kubeobj := range kubeobjs {
+		switch strings.ToLower(kubeobj.Kind) {
+		case "deployment":
+			return &kubeobj
+		}
+	}
+	return nil
+}
+
+func (deploy *NewDeploy) scaleDeployment(deplNS string, deplName string, replicas int32) error {
+	deploy.logger.Info("scaling deployment",
+		zap.String("deployment", deplName),
+		zap.String("namespace", deplNS),
+		zap.Int32("replicas", replicas))
+	_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deplNS).UpdateScale(deplName, &v1beta1.Scale{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      deplName,
+			Namespace: deplNS,
+		},
+		Spec: v1beta1.ScaleSpec{
+			Replicas: replicas,
+		},
+	})
+	return err
+}

pkg/executor/poolmgr/funcwatcher.go

@@ -0,0 +1,213 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package poolmgr
+
+import (
+	"time"
+
+	"github.com/fission/fission/pkg/types"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/utils"
+)
+
+func getIstioServiceLabels(fnName string) map[string]string {
+	return map[string]string{
+		"functionName": fnName,
+	}
+}
+
+func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset, fissionfnNamespace string, istioEnabled bool) (k8sCache.Store, k8sCache.Controller) {
+
+	resyncPeriod := 30 * time.Second
+	lw := k8sCache.NewListWatchFromClient(fissionClient.GetCrdClient(), "functions", metav1.NamespaceAll, fields.Everything())
+
+	funcStore, controller := k8sCache.NewInformer(lw, &fv1.Function{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				fn := obj.(*fv1.Function)
+
+				// Since istio only allows accessing pod through k8s service,
+				// for the functions with executor type "poolmgr" we need to
+				// create a service for sending requests to pod in pool.
+				// Functions with executor type "Newdeploy" is specialized at
+				// pod starts. In this case, just ignore such functions.
+				fnExecutorType := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
+
+				// In some cases, user may not enter the executorType explicitly, for example in his spec.yaml.
+				// we assume it to be of type poolmgr
+				if fnExecutorType != "" && fnExecutorType != fv1.ExecutorTypePoolmgr {
+					return
+				}
+
+				// create or update role-binding
+				envNs := fissionfnNamespace
+				if fn.Spec.Environment.Namespace != metav1.NamespaceDefault {
+					envNs = fn.Spec.Environment.Namespace
+				}
+
+				// TODO : Just bring to your attention during review :
+				// setup rolebinding is tried, if it fails, we dont return. we just log an error and move on, because :
+				// 1. not all functions have secrets and/or configmaps, so things will work without this rolebinding in that case.
+				// 2. on the contrary, when the route is tried, the env fetcher logs will show a 403 forbidden message and same will be relayed to executor.
+				err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.SecretConfigMapGetterRB, fn.Metadata.Namespace, types.SecretConfigMapGetterCR, types.ClusterRole, types.FissionFetcherSA, envNs)
+				if err != nil {
+					gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", types.SecretConfigMapGetterRB))
+				} else {
+					gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
+						zap.String("service_account", types.FissionFetcherSA),
+						zap.String("service_account_namepsace", envNs),
+						zap.String("function_name", fn.Metadata.Name),
+						zap.String("function_namespace", fn.Metadata.Namespace))
+				}
+
+				if istioEnabled {
+					// create a same name service for function
+					// since istio only allows the traffic to service
+					sel := map[string]string{
+						"functionName": fn.Metadata.Name,
+						"functionUid":  string(fn.Metadata.UID),
+					}
+
+					svcName := utils.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
+
+					// service for accepting user traffic
+					svc := apiv1.Service{
+						ObjectMeta: metav1.ObjectMeta{
+							Namespace: envNs,
+							Name:      svcName,
+							Labels:    getIstioServiceLabels(fn.Metadata.Name),
+						},
+						Spec: apiv1.ServiceSpec{
+							Type: apiv1.ServiceTypeClusterIP,
+							Ports: []apiv1.ServicePort{
+								// Service port name should begin with a recognized prefix, or the traffic will be
+								// treated as TCP traffic. (https://istio.io/docs/setup/kubernetes/sidecar-injection.html)
+								// Originally the ports' name are similar to "http-fetch" and "http-specialize".
+								// But for istio 0.5.1, istio-proxy return unexpected 431 error with such naming.
+								// https://github.com/istio/istio/issues/928
+								// Workaround: remove prefix
+								// TODO: prepend prefix once the bug fixed
+								{
+									Name:       "fetch",
+									Protocol:   apiv1.ProtocolTCP,
+									Port:       8000,
+									TargetPort: intstr.FromInt(8000),
+								},
+								{
+									Name:       "specialize",
+									Protocol:   apiv1.ProtocolTCP,
+									Port:       8888,
+									TargetPort: intstr.FromInt(8888),
+								},
+							},
+							Selector: sel,
+						},
+					}
+
+					// create function istio service if it does not exist
+					_, err = kubernetesClient.CoreV1().Services(envNs).Create(&svc)
+					if err != nil && !kerrors.IsAlreadyExists(err) {
+						gpm.logger.Error("error creating istio service for function",
+							zap.Error(err),
+							zap.String("service_name", svcName),
+							zap.String("function_name", fn.Metadata.Name),
+							zap.Any("selectors", sel))
+					}
+				}
+			},
+
+			DeleteFunc: func(obj interface{}) {
+				fn := obj.(*fv1.Function)
+
+				fnExecutorType := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
+				if fnExecutorType != "" && fnExecutorType != fv1.ExecutorTypePoolmgr {
+					return
+				}
+
+				envNs := fissionfnNamespace
+				if fn.Spec.Environment.Namespace != metav1.NamespaceDefault {
+					envNs = fn.Spec.Environment.Namespace
+				}
+
+				if istioEnabled {
+					svcName := utils.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
+					// delete function istio service
+					err := kubernetesClient.CoreV1().Services(envNs).Delete(svcName, nil)
+					if err != nil && !kerrors.IsNotFound(err) {
+						gpm.logger.Error("error deleting istio service for function",
+							zap.Error(err),
+							zap.String("service_name", svcName),
+							zap.String("function_name", fn.Metadata.Name))
+
+					}
+				}
+			},
+
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				oldFunc := oldObj.(*fv1.Function)
+				newFunc := newObj.(*fv1.Function)
+
+				if oldFunc.Metadata.ResourceVersion == newFunc.Metadata.ResourceVersion {
+					return
+				}
+
+				envChanged := (oldFunc.Spec.Environment.Namespace != newFunc.Spec.Environment.Namespace)
+
+				executorTypeChangedToPM := (oldFunc.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypePoolmgr &&
+					newFunc.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypePoolmgr)
+
+				// if a func's env reference gets updated and the newly referenced env is in a different ns,
+				// we need to create a rolebinding in func's ns so that the fetcher-sa in env ns has access
+				// to fetch secrets and config maps from the func's ns.
+				// similarly if executorType changed to Pool Manager, we now need a rolebinding in the func ns for fetcher sa
+				// present in env ns because for newdeploy, the fetcher sa is in function namespace
+				if envChanged || executorTypeChangedToPM {
+					envNs := fissionfnNamespace
+					if newFunc.Spec.Environment.Namespace != metav1.NamespaceDefault {
+						envNs = newFunc.Spec.Environment.Namespace
+					}
+
+					err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.SecretConfigMapGetterRB,
+						newFunc.Metadata.Namespace, types.SecretConfigMapGetterCR, types.ClusterRole,
+						types.FissionFetcherSA, envNs)
+
+					if err != nil {
+						gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", types.SecretConfigMapGetterRB))
+					} else {
+						gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
+							zap.String("service_account", types.FissionFetcherSA),
+							zap.String("service_account_namepsace", envNs),
+							zap.String("function_name", newFunc.Metadata.Name),
+							zap.String("function_namespace", newFunc.Metadata.Namespace))
+					}
+				}
+			},
+		})
+
+	return funcStore, controller
+}

pkg/executor/poolmgr/gp.go

@@ -0,0 +1,626 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package poolmgr
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/dchest/uniuri"
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+	multierror "github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/api/extensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/executor/fscache"
+	"github.com/fission/fission/pkg/executor/util"
+	fetcherClient "github.com/fission/fission/pkg/fetcher/client"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+)
+
+type (
+	GenericPool struct {
+		logger                 *zap.Logger
+		env                    *fv1.Environment
+		replicas               int32                         // num idle pods
+		deployment             *v1beta1.Deployment           // kubernetes deployment
+		namespace              string                        // namespace to keep our resources
+		functionNamespace      string                        // fallback namespace for fission functions
+		podReadyTimeout        time.Duration                 // timeout for generic pods to become ready
+		idlePodReapTime        time.Duration                 // pods unused for idlePodReapTime are deleted
+		fsCache                *fscache.FunctionServiceCache // cache funcSvc's by function, address and podname
+		useSvc                 bool                          // create k8s service for specialized pods
+		useIstio               bool
+		poolInstanceId         string           // small random string to uniquify pod names
+		runtimeImagePullPolicy apiv1.PullPolicy // pull policy for generic pool to created env deployment
+		kubernetesClient       *kubernetes.Clientset
+		fissionClient          *crd.FissionClient
+		instanceId             string // poolmgr instance id
+		labelsForPool          map[string]string
+		requestChannel         chan *choosePodRequest
+		fetcherConfig          *fetcherConfig.Config
+	}
+
+	// serialize the choosing of pods so that choices don't conflict
+	choosePodRequest struct {
+		newLabels       map[string]string
+		responseChannel chan *choosePodResponse
+	}
+	choosePodResponse struct {
+		pod *apiv1.Pod
+		error
+	}
+)
+
+func MakeGenericPool(
+	logger *zap.Logger,
+	fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset,
+	env *fv1.Environment,
+	initialReplicas int32,
+	namespace string,
+	functionNamespace string,
+	fsCache *fscache.FunctionServiceCache,
+	fetcherConfig *fetcherConfig.Config,
+	instanceId string,
+	enableIstio bool) (*GenericPool, error) {
+
+	gpLogger := logger.Named("generic_pool")
+
+	gpLogger.Info("creating pool", zap.Any("environment", env.Metadata))
+
+	// TODO: in general we need to provide the user a way to configure pools.  Initial
+	// replicas, autoscaling params, various timeouts, etc.
+	gp := &GenericPool{
+		logger:            gpLogger,
+		env:               env,
+		replicas:          initialReplicas, // TODO make this an env param instead?
+		requestChannel:    make(chan *choosePodRequest),
+		fissionClient:     fissionClient,
+		kubernetesClient:  kubernetesClient,
+		namespace:         namespace,
+		functionNamespace: functionNamespace,
+		podReadyTimeout:   5 * time.Minute, // TODO make this an env param?
+		idlePodReapTime:   3 * time.Minute, // TODO make this configurable
+		fsCache:           fsCache,
+		poolInstanceId:    uniuri.NewLen(8),
+		fetcherConfig:     fetcherConfig,
+		instanceId:        instanceId,
+		useSvc:            false,       // defaults off -- svc takes a second or more to become routable, slowing cold start
+		useIstio:          enableIstio, // defaults off -- istio integration requires pod relabeling and it takes a second or more to become routable, slowing cold start
+	}
+
+	gp.runtimeImagePullPolicy = utils.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY"))
+
+	// create fetcher SA in this ns, if not already created
+	err := fetcherConfig.SetupServiceAccount(gp.kubernetesClient, gp.namespace, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error creating fetcher service account in namespace %q", gp.namespace)
+	}
+
+	// Labels for generic deployment/RS/pods.
+	gp.labelsForPool = gp.getDeployLabels()
+
+	// create the pool
+	err = gp.createPool()
+	if err != nil {
+		return nil, err
+	}
+	gpLogger.Info("deployment created", zap.Any("environment", env.Metadata))
+
+	go gp.choosePodService()
+
+	return gp, nil
+}
+
+func (gp *GenericPool) getDeployLabels() map[string]string {
+	return map[string]string{
+		fv1.EXECUTOR_INSTANCEID_LABEL: gp.instanceId,
+		types.EXECUTOR_TYPE:           fv1.ExecutorTypePoolmgr,
+		types.ENVIRONMENT_NAME:        gp.env.Metadata.Name,
+		types.ENVIRONMENT_NAMESPACE:   gp.env.Metadata.Namespace,
+		types.ENVIRONMENT_UID:         string(gp.env.Metadata.UID),
+		"managed":                     "true", // this allows us to easily find pods managed by the deployment
+	}
+}
+
+// choosePodService serializes the choosing of pods
+func (gp *GenericPool) choosePodService() {
+	for {
+		select {
+		case req := <-gp.requestChannel:
+			pod, err := gp._choosePod(req.newLabels)
+			if err != nil {
+				req.responseChannel <- &choosePodResponse{error: err}
+				continue
+			}
+			req.responseChannel <- &choosePodResponse{pod: pod}
+		}
+	}
+}
+
+// choosePod picks a ready pod from the pool and relabels it, waiting if necessary.
+// returns the pod API object.
+func (gp *GenericPool) choosePod(newLabels map[string]string) (*apiv1.Pod, error) {
+	req := &choosePodRequest{
+		newLabels:       newLabels,
+		responseChannel: make(chan *choosePodResponse),
+	}
+	gp.requestChannel <- req
+	resp := <-req.responseChannel
+	return resp.pod, resp.error
+}
+
+// _choosePod is called serially by choosePodService
+func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, error) {
+	startTime := time.Now()
+	for {
+		// Retries took too long, error out.
+		if time.Since(startTime) > gp.podReadyTimeout {
+			gp.logger.Error("timed out waiting for pod", zap.Any("labels", newLabels), zap.Duration("timeout", gp.podReadyTimeout))
+			return nil, errors.New("timeout: waited too long to get a ready pod")
+		}
+
+		// Get pods; filter the ones that are ready
+		podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(
+			metav1.ListOptions{
+				LabelSelector: labels.Set(
+					gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
+			})
+		if err != nil {
+			return nil, err
+		}
+		readyPods := make([]*apiv1.Pod, 0, len(podList.Items))
+		for i := range podList.Items {
+			pod := podList.Items[i]
+
+			// Ignore not ready pod here
+			if !utils.IsReadyPod(&pod) {
+				continue
+			}
+
+			// add it to the list of ready pods
+			readyPods = append(readyPods, &pod)
+		}
+		gp.logger.Info("found ready pods",
+			zap.Any("labels", newLabels),
+			zap.Int("ready_count", len(readyPods)),
+			zap.Int("total", len(podList.Items)))
+
+		// If there are no ready pods, wait and retry.
+		if len(readyPods) == 0 {
+			err = gp.waitForReadyPod()
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+
+		// Pick a ready pod.  For now just choose randomly;
+		// ideally we'd care about which node it's running on,
+		// and make a good scheduling decision.
+		chosenPod := readyPods[rand.Intn(len(readyPods))]
+
+		if gp.env.Spec.AllowedFunctionsPerContainer != types.AllowedFunctionsPerContainerInfinite {
+			// Relabel.  If the pod already got picked and
+			// modified, this should fail; in that case just
+			// retry.
+			chosenPod.ObjectMeta.Labels = newLabels
+			gp.logger.Info("relabeling pod", zap.String("pod", chosenPod.ObjectMeta.Name))
+			_, err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Update(chosenPod)
+			if err != nil {
+				gp.logger.Error("failed to relabel pod", zap.Error(err), zap.String("pod", chosenPod.ObjectMeta.Name))
+				continue
+			}
+		}
+		gp.logger.Info("chose pod", zap.String("pod", chosenPod.ObjectMeta.Name), zap.Duration("elapsed_time", time.Since(startTime)))
+		return chosenPod, nil
+	}
+}
+
+func (gp *GenericPool) labelsForFunction(metadata *metav1.ObjectMeta) map[string]string {
+	label := gp.getDeployLabels()
+	label[types.FUNCTION_NAME] = metadata.Name
+	label[types.FUNCTION_UID] = string(metadata.UID)
+	label[types.FUNCTION_NAMESPACE] = metadata.Namespace // function CRD must stay within same namespace of environment CRD
+	label["managed"] = "false"                           // this allows us to easily find pods not managed by the deployment
+	return label
+
+}
+
+func (gp *GenericPool) scheduleDeletePod(name string) {
+	go func() {
+		// The sleep allows debugging or collecting logs from the pod before it's
+		// cleaned up.  (We need a better solutions for both those things; log
+		// aggregation and storage will help.)
+		gp.logger.Error("error in pod - scheduling cleanup", zap.String("pod", name))
+		// Ignore sleep here if istio feature is enabled, function pod
+		// will be deleted after 6 mins (terminationGracePeriodSeconds).
+		if !gp.useIstio {
+			time.Sleep(5 * time.Minute)
+		}
+		gp.kubernetesClient.CoreV1().Pods(gp.namespace).Delete(name, nil)
+	}()
+}
+
+func IsIPv6(podIP string) bool {
+	ip := net.ParseIP(podIP)
+	return ip != nil && strings.Contains(podIP, ":")
+}
+
+func (gp *GenericPool) getSpecializeUrl(podIP string) string {
+	testUrl := os.Getenv("TEST_SPECIALIZE_URL")
+	if len(testUrl) != 0 {
+		// it takes a second or so for the test service to
+		// become routable once a pod is relabeled. This is
+		// super hacky, but only runs in unit tests.
+		time.Sleep(5 * time.Second)
+		return testUrl
+	}
+	isv6 := IsIPv6(podIP)
+	var baseUrl string
+	if isv6 == false {
+		baseUrl = fmt.Sprintf("http://%v:8000/", podIP)
+	} else if isv6 == true { // We use bracket if the IP is in IPv6.
+		baseUrl = fmt.Sprintf("http://[%v]:8000/", podIP)
+	}
+	return baseUrl
+
+}
+
+// specializePod chooses a pod, copies the required user-defined function to that pod
+// (via fetcher), and calls the function-run container to load it, resulting in a
+// specialized pod.
+func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metadata *metav1.ObjectMeta) error {
+	// for fetcher we don't need to create a service, just talk to the pod directly
+	podIP := pod.Status.PodIP
+	if len(podIP) == 0 {
+		return errors.Errorf("Pod %s in namespace %s has no IP", pod.ObjectMeta.Name, pod.ObjectMeta.Namespace)
+	}
+	// specialize pod with service
+	if gp.useIstio {
+		svc := utils.GetFunctionIstioServiceName(metadata.Name, metadata.Namespace)
+		podIP = fmt.Sprintf("%v.%v", svc, gp.namespace)
+	}
+
+	// tell fetcher to get the function.
+	fetcherUrl := gp.getSpecializeUrl(podIP)
+	gp.logger.Info("calling fetcher to copy function", zap.String("function", metadata.Name), zap.String("url", fetcherUrl))
+
+	fn, err := gp.fissionClient.
+		Functions(metadata.Namespace).
+		Get(metadata.Name)
+	if err != nil {
+		return err
+	}
+
+	specializeReq := gp.fetcherConfig.NewSpecializeRequest(fn, gp.env)
+
+	gp.logger.Info("specializing pod", zap.String("function", metadata.Name))
+
+	err = fetcherClient.MakeClient(gp.logger, fetcherUrl).Specialize(ctx, &specializeReq)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// getPoolName returns a unique name of an environment
+func (gp *GenericPool) getPoolName() string {
+	return strings.ToLower(fmt.Sprintf("poolmgr-%v-%v-%v", gp.env.Metadata.Name, gp.env.Metadata.Namespace, uniuri.NewLen(8)))
+}
+
+// A pool is a deployment of generic containers for an env.  This
+// creates the pool but doesn't wait for any pods to be ready.
+func (gp *GenericPool) createPool() error {
+	// Use long terminationGracePeriodSeconds for connection draining in case that
+	// pod still runs user functions.
+	gracePeriodSeconds := int64(6 * 60)
+	if gp.env.Spec.TerminationGracePeriod > 0 {
+		gracePeriodSeconds = gp.env.Spec.TerminationGracePeriod
+	}
+
+	podAnnotations := gp.env.Metadata.Annotations
+	if podAnnotations == nil {
+		podAnnotations = make(map[string]string)
+	}
+	if gp.useIstio && gp.env.Spec.AllowAccessToExternalNetwork {
+		podAnnotations["sidecar.istio.io/inject"] = "false"
+	}
+
+	deployment := &v1beta1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   gp.getPoolName(),
+			Labels: gp.labelsForPool,
+		},
+		Spec: v1beta1.DeploymentSpec{
+			Replicas: &gp.replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: gp.labelsForPool,
+			},
+			Template: apiv1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      gp.labelsForPool,
+					Annotations: podAnnotations,
+				},
+				Spec: apiv1.PodSpec{
+					Containers: []apiv1.Container{
+						util.MergeContainerSpecs(&apiv1.Container{
+							Name:                   gp.env.Metadata.Name,
+							Image:                  gp.env.Spec.Runtime.Image,
+							ImagePullPolicy:        gp.runtimeImagePullPolicy,
+							TerminationMessagePath: "/dev/termination-log",
+							Resources:              gp.env.Spec.Resources,
+							// Pod is removed from endpoints list for service when it's
+							// state became "Termination". We used preStop hook as the
+							// workaround for connection draining since pod maybe shutdown
+							// before grace period expires.
+							// https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods
+							// https://github.com/kubernetes/kubernetes/issues/47576#issuecomment-308900172
+							Lifecycle: &apiv1.Lifecycle{
+								PreStop: &apiv1.Handler{
+									Exec: &apiv1.ExecAction{
+										Command: []string{
+											"/bin/sleep",
+											fmt.Sprintf("%v", gracePeriodSeconds),
+										},
+									},
+								},
+							},
+						}, gp.env.Spec.Runtime.Container),
+					},
+					ServiceAccountName: "fission-fetcher",
+					// TerminationGracePeriodSeconds should be equal to the
+					// sleep time of preStop to make sure that SIGTERM is sent
+					// to pod after 6 mins.
+					TerminationGracePeriodSeconds: &gracePeriodSeconds,
+				},
+			},
+		},
+	}
+
+	// Order of merging is important here - first fetcher, then containers and lastly pod spec
+	err := gp.fetcherConfig.AddFetcherToPodSpec(&deployment.Spec.Template.Spec, gp.env.Metadata.Name)
+	if err != nil {
+		return err
+	}
+
+	if gp.env.Spec.Runtime.PodSpec != nil {
+		err = util.MergePodSpec(&deployment.Spec.Template.Spec, gp.env.Spec.Runtime.PodSpec)
+		if err != nil {
+			return err
+		}
+	}
+
+	depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment)
+	if err != nil {
+		gp.logger.Error("error creating deployment in kubernetes", zap.Error(err), zap.String("deployment", deployment.Name))
+		return err
+	}
+	gp.deployment = depl
+	return nil
+}
+
+func (gp *GenericPool) waitForReadyPod() error {
+	startTime := time.Now()
+	for {
+		// TODO: for now we just poll; use a watch instead
+		depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get(
+			gp.deployment.ObjectMeta.Name, metav1.GetOptions{})
+		if err != nil {
+			e := "error waiting for ready pod for deployment"
+			gp.logger.Error(e, zap.String("deployment", gp.deployment.ObjectMeta.Name), zap.String("namespace", gp.namespace))
+			return fmt.Errorf("%s %q in namespace %q", e, gp.deployment.ObjectMeta.Name, gp.namespace)
+		}
+
+		gp.deployment = depl
+		if gp.deployment.Status.AvailableReplicas > 0 {
+			return nil
+		}
+
+		if time.Since(startTime) > gp.podReadyTimeout {
+			podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(metav1.ListOptions{
+				LabelSelector: labels.Set(
+					gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
+			})
+			if err != nil {
+				gp.logger.Error("error getting pod list after timeout waiting for ready pod", zap.Error(err))
+			}
+
+			// Since even single pod is not ready, choosing the first pod to inspect is a good approximation. In future this can be done better
+			pod := podList.Items[0]
+			var multierr *multierror.Error
+			for _, cStatus := range pod.Status.ContainerStatuses {
+				if cStatus.Ready != true {
+					multierr = multierror.Append(multierr, errors.New(fmt.Sprintf("%v: %v", cStatus.State.Waiting.Reason, cStatus.State.Waiting.Message)))
+				}
+			}
+			return errors.Wrapf(multierr, "Timeout: waited too long for pod of deployment %v in namespace %v to be ready",
+				gp.deployment.ObjectMeta.Name, gp.namespace)
+		}
+		time.Sleep(1000 * time.Millisecond)
+	}
+}
+
+func (gp *GenericPool) createSvc(name string, labels map[string]string) (*apiv1.Service, error) {
+	service := apiv1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: apiv1.ServiceSpec{
+			Type: apiv1.ServiceTypeClusterIP,
+			Ports: []apiv1.ServicePort{
+				{
+					Protocol:   apiv1.ProtocolTCP,
+					Port:       80,
+					TargetPort: intstr.FromInt(8888),
+				},
+			},
+			Selector: labels,
+		},
+	}
+	svc, err := gp.kubernetesClient.CoreV1().Services(gp.namespace).Create(&service)
+	return svc, err
+}
+
+func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
+	gp.logger.Info("choosing pod from pool", zap.String("function", m.Name))
+	newLabels := gp.labelsForFunction(m)
+
+	if gp.useIstio {
+		// Istio only allows accessing pod through k8s service, and requests come to
+		// service are not always being routed to the same pod. For example:
+
+		// If there is only one pod (podA) behind the service svcX.
+
+		// svcX -> podA
+
+		// All requests (specialize request & function access requests)
+		// will be routed to podA without any problem.
+
+		// If podA and podB are behind svcX.
+
+		// svcX -> podA (specialized)
+		//      -> podB (non-specialized)
+
+		// The specialize request may be routed to podA and the function access
+		// requests may go to podB. In this case, the function cannot be served
+		// properly.
+
+		// To prevent such problem, we need to delete old versions function pods
+		// and make sure that there is only one pod behind the service
+
+		sel := map[string]string{
+			"functionName": m.Name,
+			"functionUid":  string(m.UID),
+		}
+		podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(metav1.ListOptions{
+			LabelSelector: labels.Set(sel).AsSelector().String(),
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// Remove old versions function pods
+		for _, pod := range podList.Items {
+			// Delete pod no matter what status it is
+			gp.kubernetesClient.CoreV1().Pods(gp.namespace).Delete(pod.ObjectMeta.Name, nil)
+		}
+	}
+
+	pod, err := gp.choosePod(newLabels)
+	if err != nil {
+		return nil, err
+	}
+
+	err = gp.specializePod(ctx, pod, m)
+	if err != nil {
+		gp.scheduleDeletePod(pod.ObjectMeta.Name)
+		return nil, err
+	}
+	gp.logger.Info("specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
+
+	var svcHost string
+	if gp.useSvc && !gp.useIstio {
+		svcName := fmt.Sprintf("svc-%v", m.Name)
+		if len(m.UID) > 0 {
+			svcName = fmt.Sprintf("%s-%v", svcName, m.UID)
+		}
+
+		labels := gp.labelsForFunction(m)
+		svc, err := gp.createSvc(svcName, labels)
+		if err != nil {
+			gp.scheduleDeletePod(pod.ObjectMeta.Name)
+			return nil, err
+		}
+		if svc.ObjectMeta.Name != svcName {
+			gp.scheduleDeletePod(pod.ObjectMeta.Name)
+			return nil, errors.Errorf("sanity check failed for svc %v", svc.ObjectMeta.Name)
+		}
+
+		// the fission router isn't in the same namespace, so return a
+		// namespace-qualified hostname
+		svcHost = fmt.Sprintf("%v.%v", svcName, gp.namespace)
+	} else if gp.useIstio {
+		svc := utils.GetFunctionIstioServiceName(m.Name, m.Namespace)
+		svcHost = fmt.Sprintf("%v.%v:8888", svc, gp.namespace)
+	} else {
+		gp.logger.Info("using pod IP for specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
+		svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP)
+	}
+
+	kubeObjRefs := []apiv1.ObjectReference{
+		{
+			Kind:            "pod",
+			Name:            pod.ObjectMeta.Name,
+			APIVersion:      pod.TypeMeta.APIVersion,
+			Namespace:       pod.ObjectMeta.Namespace,
+			ResourceVersion: pod.ObjectMeta.ResourceVersion,
+			UID:             pod.ObjectMeta.UID,
+		},
+	}
+
+	fsvc := &fscache.FuncSvc{
+		Name:              pod.ObjectMeta.Name,
+		Function:          m,
+		Environment:       gp.env,
+		Address:           svcHost,
+		KubernetesObjects: kubeObjRefs,
+		Executor:          fscache.POOLMGR,
+		Ctime:             time.Now(),
+		Atime:             time.Now(),
+	}
+
+	_, err = gp.fsCache.Add(*fsvc)
+	if err != nil {
+		return nil, err
+	}
+	return fsvc, nil
+}
+
+// destroys the pool -- the deployment, replicaset and pods
+func (gp *GenericPool) destroy() error {
+	deletePropagation := metav1.DeletePropagationBackground
+	delOpt := metav1.DeleteOptions{
+		PropagationPolicy: &deletePropagation,
+	}
+	err := gp.kubernetesClient.ExtensionsV1beta1().
+		Deployments(gp.namespace).Delete(gp.deployment.ObjectMeta.Name, &delOpt)
+	if err != nil {
+		gp.logger.Error("error destroying deployment",
+			zap.Error(err),
+			zap.String("deployment_name", gp.deployment.ObjectMeta.Name),
+			zap.String("deployment_namespace", gp.namespace))
+		return err
+	}
+	return nil
+}

pkg/executor/poolmgr/gpm.go

@@ -0,0 +1,368 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package poolmgr
+
+import (
+	"context"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/fission/fission/pkg/utils"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8sTypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/cache"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/executor/fscache"
+	"github.com/fission/fission/pkg/executor/reaper"
+	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
+	"github.com/fission/fission/pkg/types"
+)
+
+type requestType int
+
+const (
+	GET_POOL requestType = iota
+	CLEANUP_POOLS
+)
+
+type (
+	GenericPoolManager struct {
+		logger *zap.Logger
+
+		pools            map[string]*GenericPool
+		kubernetesClient *kubernetes.Clientset
+		namespace        string
+
+		fissionClient  *crd.FissionClient
+		functionEnv    *cache.Cache
+		fsCache        *fscache.FunctionServiceCache
+		instanceId     string
+		requestChannel chan *request
+
+		enableIstio   bool
+		fetcherConfig *fetcherConfig.Config
+
+		funcStore      k8sCache.Store
+		funcController k8sCache.Controller
+		pkgStore       k8sCache.Store
+		pkgController  k8sCache.Controller
+
+		idlePodReapTime time.Duration
+	}
+	request struct {
+		requestType
+		env             *fv1.Environment
+		envList         []fv1.Environment
+		responseChannel chan *response
+	}
+	response struct {
+		error
+		pool *GenericPool
+	}
+)
+
+func MakeGenericPoolManager(
+	logger *zap.Logger,
+	fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset,
+	functionNamespace string,
+	fetcherConfig *fetcherConfig.Config,
+	instanceId string) *GenericPoolManager {
+
+	gpmLogger := logger.Named("generic_pool_manager")
+
+	gpm := &GenericPoolManager{
+		logger:           gpmLogger,
+		pools:            make(map[string]*GenericPool),
+		kubernetesClient: kubernetesClient,
+		namespace:        functionNamespace,
+		fissionClient:    fissionClient,
+		functionEnv:      cache.MakeCache(10*time.Second, 0),
+		fsCache:          fscache.MakeFunctionServiceCache(gpmLogger),
+		instanceId:       instanceId,
+		requestChannel:   make(chan *request),
+		idlePodReapTime:  2 * time.Minute,
+		fetcherConfig:    fetcherConfig,
+	}
+	go gpm.service()
+	go gpm.eagerPoolCreator()
+
+	if len(os.Getenv("ENABLE_ISTIO")) > 0 {
+		istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
+		if err != nil {
+			gpmLogger.Info("failed to parse ENABLE_ISTIO")
+		}
+		gpm.enableIstio = istio
+	}
+
+	gpm.funcStore, gpm.funcController = gpm.makeFuncController(
+		gpm.fissionClient, gpm.kubernetesClient, gpm.namespace, gpm.enableIstio)
+
+	gpm.pkgStore, gpm.pkgController = gpm.makePkgController(gpm.fissionClient, gpm.kubernetesClient, gpm.namespace)
+
+	return gpm
+}
+
+func (gpm *GenericPoolManager) Run(ctx context.Context) {
+	go gpm.funcController.Run(ctx.Done())
+	go gpm.pkgController.Run(ctx.Done())
+	go gpm.idleObjectReaper()
+}
+
+func (gpm *GenericPoolManager) service() {
+	for {
+		req := <-gpm.requestChannel
+		switch req.requestType {
+		case GET_POOL:
+			// just because they are missing in the cache, we end up creating another duplicate pool.
+			var err error
+			pool, ok := gpm.pools[crd.CacheKey(&req.env.Metadata)]
+			if !ok {
+				poolsize := gpm.getEnvPoolsize(req.env)
+				switch req.env.Spec.AllowedFunctionsPerContainer {
+				case types.AllowedFunctionsPerContainerInfinite:
+					poolsize = 1
+				}
+
+				// To support backward compatibility, if envs are created in default ns, we go ahead
+				// and create pools in fission-function ns as earlier.
+				ns := gpm.namespace
+				if req.env.Metadata.Namespace != metav1.NamespaceDefault {
+					ns = req.env.Metadata.Namespace
+				}
+
+				pool, err = MakeGenericPool(gpm.logger,
+					gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize,
+					ns, gpm.namespace, gpm.fsCache, gpm.fetcherConfig, gpm.instanceId, gpm.enableIstio)
+				if err != nil {
+					req.responseChannel <- &response{error: err}
+					continue
+				}
+				gpm.pools[crd.CacheKey(&req.env.Metadata)] = pool
+			}
+			req.responseChannel <- &response{pool: pool}
+		case CLEANUP_POOLS:
+			latestEnvPoolsize := make(map[string]int)
+			for _, env := range req.envList {
+				latestEnvPoolsize[crd.CacheKey(&env.Metadata)] = int(gpm.getEnvPoolsize(&env))
+			}
+			for key, pool := range gpm.pools {
+				poolsize, ok := latestEnvPoolsize[key]
+				if !ok || poolsize == 0 {
+					// Env no longer exists or pool size changed to zero
+
+					gpm.logger.Info("destroying generic pool", zap.Any("environment", pool.env.Metadata))
+					delete(gpm.pools, key)
+
+					// and delete the pool asynchronously.
+					go pool.destroy()
+				}
+			}
+			// no response, caller doesn't wait
+		}
+	}
+}
+
+func (gpm *GenericPoolManager) GetPool(env *fv1.Environment) (*GenericPool, error) {
+	c := make(chan *response)
+	gpm.requestChannel <- &request{
+		requestType:     GET_POOL,
+		env:             env,
+		responseChannel: c,
+	}
+	resp := <-c
+	return resp.pool, resp.error
+}
+
+func (gpm *GenericPoolManager) CleanupPools(envs []fv1.Environment) {
+	gpm.requestChannel <- &request{
+		requestType: CLEANUP_POOLS,
+		envList:     envs,
+	}
+}
+
+func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
+	// from Func -> get Env
+	gpm.logger.Info("getting environment for function", zap.String("function", metadata.Name))
+	env, err := gpm.getFunctionEnv(metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	pool, err := gpm.GetPool(env)
+	if err != nil {
+		return nil, err
+	}
+	// from GenericPool -> get one function container
+	// (this also adds to the cache)
+	gpm.logger.Info("getting function service from pool", zap.String("function", metadata.Name))
+	return pool.GetFuncSvc(ctx, metadata)
+}
+
+func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*fv1.Environment, error) {
+	var env *fv1.Environment
+
+	// Cached ?
+	result, err := gpm.functionEnv.Get(crd.CacheKey(m))
+	if err == nil {
+		env = result.(*fv1.Environment)
+		return env, nil
+	}
+
+	// Cache miss -- get func from controller
+	f, err := gpm.fissionClient.Functions(m.Namespace).Get(m.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get env from metadata
+	gpm.logger.Info("getting env", zap.Any("function", m))
+	env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// cache for future lookups
+	gpm.functionEnv.Set(crd.CacheKey(m), env)
+
+	return env, nil
+}
+
+func (gpm *GenericPoolManager) eagerPoolCreator() {
+	pollSleep := time.Duration(2 * time.Second)
+	for {
+		// get list of envs from controller
+		envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
+		if err != nil {
+			if utils.IsNetworkError(err) {
+				gpm.logger.Error("encountered network error, retrying", zap.Error(err))
+				time.Sleep(5 * time.Second)
+				continue
+			}
+			gpm.logger.Fatal("failed to get environment list", zap.Error(err))
+		}
+
+		// Create pools for all envs.  TODO: we should make this a bit less eager, only
+		// creating pools for envs that are actually used by functions.  Also we might want
+		// to keep these eagerly created pools smaller than the ones created when there are
+		// actual function calls.
+		for i := range envs.Items {
+			env := envs.Items[i]
+			// Create pool only if poolsize greater than zero
+			if gpm.getEnvPoolsize(&env) > 0 {
+				_, err := gpm.GetPool(&envs.Items[i])
+				if err != nil {
+					gpm.logger.Error("eager-create pool failed", zap.Error(err))
+				}
+			}
+		}
+
+		// Clean up pools whose env was deleted
+		gpm.CleanupPools(envs.Items)
+		time.Sleep(pollSleep)
+	}
+}
+
+func (gpm *GenericPoolManager) getEnvPoolsize(env *fv1.Environment) int32 {
+	var poolsize int32
+	if env.Spec.Version < 3 {
+		poolsize = 3
+	} else {
+		poolsize = int32(env.Spec.Poolsize)
+	}
+	return poolsize
+}
+
+// IsValid checks if pod is not deleted and that it has the address passed as the argument. Also checks that all the
+// containers in it are reporting a ready status for the healthCheck.
+func (gpm *GenericPoolManager) IsValid(fsvc *fscache.FuncSvc) bool {
+	for _, obj := range fsvc.KubernetesObjects {
+		if obj.Kind == "pod" {
+			pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{})
+			if err == nil && strings.Contains(fsvc.Address, pod.Status.PodIP) && utils.IsReadyPod(pod) {
+				gpm.logger.Info("valid pod address", zap.String("address", fsvc.Address))
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// idleObjectReaper reaps objects after certain idle time
+func (gpm *GenericPoolManager) idleObjectReaper() {
+
+	pollSleep := time.Duration(gpm.idlePodReapTime)
+	for {
+		time.Sleep(pollSleep)
+
+		envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
+		if err != nil {
+			gpm.logger.Fatal("failed to get environment list", zap.Error(err))
+		}
+
+		envList := make(map[k8sTypes.UID]struct{})
+		for _, env := range envs.Items {
+			envList[env.Metadata.UID] = struct{}{}
+		}
+
+		funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime)
+		if err != nil {
+			gpm.logger.Error("error reaping idle pods", zap.Error(err))
+			continue
+		}
+
+		for _, fsvc := range funcSvcs {
+			if fsvc.Executor != fscache.POOLMGR {
+				continue
+			}
+
+			// For function with the environment that no longer exists, executor
+			// cleanups the idle pod as usual and prints log to notify user.
+			if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
+				gpm.logger.Info("function environment no longer exists",
+					zap.String("environment", fsvc.Environment.Metadata.Name),
+					zap.String("function", fsvc.Name))
+			}
+
+			if fsvc.Environment.Spec.AllowedFunctionsPerContainer == types.AllowedFunctionsPerContainerInfinite {
+				continue
+			}
+
+			deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime)
+			if err != nil {
+				gpm.logger.Error("error deleting Kubernetes objects for function service",
+					zap.Error(err),
+					zap.Any("service", fsvc))
+			}
+
+			if !deleted {
+				continue
+			}
+
+			for _, kubeobj := range fsvc.KubernetesObjects {
+				reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &kubeobj)
+			}
+		}
+	}
+}

pkg/executor/poolmgr/packagewatcher.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package poolmgr
+
+import (
+	"time"
+
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	k8sCache "k8s.io/client-go/tools/cache"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+)
+
+// TODO : It may make sense to make each of add, update, delete funcs run as separate go routines.
+func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClient,
+	kubernetesClient *kubernetes.Clientset, fissionfnNamespace string) (k8sCache.Store, k8sCache.Controller) {
+
+	resyncPeriod := 30 * time.Second
+	lw := k8sCache.NewListWatchFromClient(fissionClient.GetCrdClient(), "packages", metav1.NamespaceAll, fields.Everything())
+	pkgStore, controller := k8sCache.NewInformer(lw, &fv1.Package{}, resyncPeriod,
+		k8sCache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				pkg := obj.(*fv1.Package)
+				gpm.logger.Debug("list watch for package reported a new package addition",
+					zap.String("package_name", pkg.Metadata.Name),
+					zap.String("package_namepsace", pkg.Metadata.Namespace))
+
+				// create or update role-binding for fetcher sa in env ns to be able to get the pkg contents from pkg namespace
+				envNs := fissionfnNamespace
+				if pkg.Spec.Environment.Namespace != metav1.NamespaceDefault {
+					envNs = pkg.Spec.Environment.Namespace
+				}
+
+				// here, we return if we hit an error during rolebinding setup. this is because this rolebinding is mandatory for
+				// every function's package to be loaded into its env. without that, there's no point to move forward.
+				err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.PackageGetterRB, pkg.Metadata.Namespace, types.PackageGetterCR, types.ClusterRole, types.FissionFetcherSA, envNs)
+				if err != nil {
+					gpm.logger.Error("error creating rolebinding for package",
+						zap.Error(err),
+						zap.String("role_binding", types.PackageGetterRB),
+						zap.String("package_name", pkg.Metadata.Name),
+						zap.String("package_namespace", pkg.Metadata.Namespace))
+					return
+				}
+
+				gpm.logger.Debug("successfully set up rolebinding for fetcher service account",
+					zap.String("service_account", types.FissionFetcherSA),
+					zap.String("service_account_namespace", envNs),
+					zap.String("package_name", pkg.Metadata.Name),
+					zap.String("package_namespace", pkg.Metadata.Namespace))
+			},
+
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				oldPkg := oldObj.(*fv1.Package)
+				newPkg := newObj.(*fv1.Package)
+
+				if oldPkg.Metadata.ResourceVersion == newPkg.Metadata.ResourceVersion {
+					return
+				}
+
+				// if a pkg's env reference gets updated and the newly referenced env is in a different ns,
+				// we need to update the role-binding in pkg ns to grant permissions to the fetcher-sa in env ns
+				// to do a get on pkg
+				if oldPkg.Spec.Environment.Namespace != newPkg.Spec.Environment.Namespace {
+					envNs := fissionfnNamespace
+					if newPkg.Spec.Environment.Namespace != metav1.NamespaceDefault {
+						envNs = newPkg.Spec.Environment.Namespace
+					}
+
+					err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.PackageGetterRB,
+						newPkg.Metadata.Namespace, types.PackageGetterCR, types.ClusterRole,
+						types.FissionFetcherSA, envNs)
+					if err != nil {
+						gpm.logger.Error("error updating rolebinding for package",
+							zap.Error(err),
+							zap.String("role_binding", types.PackageGetterRB),
+							zap.String("package_name", newPkg.Metadata.Name),
+							zap.String("package_namespace", newPkg.Metadata.Namespace))
+						return
+					}
+
+					gpm.logger.Debug("successfully updated rolebinding for fetcher service account",
+						zap.String("service_account", types.FissionFetcherSA),
+						zap.String("service_account_namespace", envNs),
+						zap.String("package_name", newPkg.Metadata.Name),
+						zap.String("package_namespace", newPkg.Metadata.Namespace))
+				}
+			},
+		})
+
+	return pkgStore, controller
+}

pkg/executor/reaper/reaper.go

@@ -0,0 +1,371 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package reaper
+
+import (
+	"strings"
+	"time"
+
+	"github.com/fission/fission/pkg/utils"
+	"go.uber.org/zap"
+	apiv1 "k8s.io/api/core/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/types"
+)
+
+var (
+	deletePropagation = meta_v1.DeletePropagationBackground
+	delOpt            = meta_v1.DeleteOptions{PropagationPolicy: &deletePropagation}
+)
+
+// CleanupOldExecutorObjects cleans up resources created by old executor instances
+func CleanupOldExecutorObjects(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, instanceId string) {
+	go func() {
+		err := cleanup(logger, kubernetesClient, instanceId)
+		if err != nil {
+			// TODO retry reaper; logged and ignored for now
+			logger.Error("Failed to cleanup old executor objects", zap.Error(err))
+		}
+	}()
+}
+
+func cleanup(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
+
+	err := cleanupServices(logger, client, instanceId)
+	if err != nil {
+		return err
+	}
+
+	err = cleanupHpa(logger, client, instanceId)
+	if err != nil {
+		return err
+	}
+
+	// Deployments are used for idle pools and can be cleaned up
+	// immediately.  (We should "adopt" these instead of creating
+	// a new pool.)
+	err = cleanupDeployments(logger, client, instanceId)
+	if err != nil {
+		return err
+	}
+
+	// Pods might still be running user functions, so we give them
+	// a few minutes before terminating them.  This time is the
+	// maximum function runtime, plus the time a router might
+	// still route to an old instance, i.e. router cache expiry
+	// time.
+	time.Sleep(6 * time.Minute)
+
+	err = cleanupPods(logger, client, instanceId)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CleanupKubeObject deletes given kubernetes object
+func CleanupKubeObject(logger *zap.Logger, kubeClient *kubernetes.Clientset, kubeobj *apiv1.ObjectReference) {
+	switch strings.ToLower(kubeobj.Kind) {
+	case "pod":
+		err := kubeClient.CoreV1().Pods(kubeobj.Namespace).Delete(kubeobj.Name, nil)
+		if err != nil {
+			logger.Error("error cleaning up pod", zap.Error(err), zap.String("pod", kubeobj.Name))
+		}
+
+	case "service":
+		err := kubeClient.CoreV1().Services(kubeobj.Namespace).Delete(kubeobj.Name, nil)
+		if err != nil {
+			logger.Error("error cleaning up service", zap.Error(err), zap.String("service", kubeobj.Name))
+		}
+
+	case "deployment":
+		err := kubeClient.ExtensionsV1beta1().Deployments(kubeobj.Namespace).Delete(kubeobj.Name, &delOpt)
+		if err != nil {
+			logger.Error("error cleaning up deployment", zap.Error(err), zap.String("deployment", kubeobj.Name))
+		}
+
+	case "horizontalpodautoscaler":
+		err := kubeClient.AutoscalingV1().HorizontalPodAutoscalers(kubeobj.Namespace).Delete(kubeobj.Name, nil)
+		if err != nil {
+			logger.Error("error cleaning up horizontalpodautoscaler", zap.Error(err), zap.String("horizontalpodautoscaler", kubeobj.Name))
+		}
+
+	default:
+		logger.Error("Could not identifying the object type to clean up", zap.String("type", kubeobj.Kind), zap.Any("object", kubeobj))
+
+	}
+}
+
+func cleanupDeployments(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
+	deploymentList, err := client.ExtensionsV1beta1().Deployments(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, dep := range deploymentList.Items {
+		id, ok := dep.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
+		if ok && id != instanceId {
+			logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
+			err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
+			if err != nil {
+				logger.Error("error cleaning up deployment",
+					zap.Error(err),
+					zap.String("deployment_name", dep.ObjectMeta.Name),
+					zap.String("deployment_namespace", dep.ObjectMeta.Namespace))
+			}
+			// ignore err
+		}
+		// Backward compatibility with older label name
+		pid, pok := dep.ObjectMeta.Labels[types.POOLMGR_INSTANCEID_LABEL]
+		if pok && pid != instanceId {
+			logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
+			err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
+			if err != nil {
+				logger.Error("error cleaning up deployment",
+					zap.Error(err),
+					zap.String("deployment_name", dep.ObjectMeta.Name),
+					zap.String("deployment_namespace", dep.ObjectMeta.Namespace))
+			}
+			// ignore err
+		}
+	}
+	return nil
+}
+
+func cleanupPods(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
+	podList, err := client.CoreV1().Pods(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, pod := range podList.Items {
+		id, ok := pod.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
+		if ok && id != instanceId {
+			logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
+			err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
+			if err != nil {
+				logger.Error("error cleaning up pod",
+					zap.Error(err),
+					zap.String("pod_name", pod.ObjectMeta.Name),
+					zap.String("pod_namespace", pod.ObjectMeta.Namespace))
+			}
+			// ignore err
+		}
+		// Backward compatibility with older label name
+		pid, pok := pod.ObjectMeta.Labels[types.POOLMGR_INSTANCEID_LABEL]
+		if pok && pid != instanceId {
+			logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
+			err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
+			if err != nil {
+				logger.Error("error cleaning up pod",
+					zap.Error(err),
+					zap.String("pod_name", pod.ObjectMeta.Name),
+					zap.String("pod_namespace", pod.ObjectMeta.Namespace))
+			}
+			// ignore err
+		}
+
+	}
+	return nil
+}
+
+func cleanupServices(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
+	svcList, err := client.CoreV1().Services(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, svc := range svcList.Items {
+		id, ok := svc.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
+		if ok && id != instanceId {
+			logger.Info("cleaning up service", zap.String("service", svc.ObjectMeta.Name))
+			err := client.CoreV1().Services(svc.ObjectMeta.Namespace).Delete(svc.ObjectMeta.Name, nil)
+			if err != nil {
+				logger.Error("error cleaning up service",
+					zap.Error(err),
+					zap.String("service_name", svc.ObjectMeta.Name),
+					zap.String("service_namespace", svc.ObjectMeta.Namespace))
+			}
+			// ignore err
+		}
+	}
+	return nil
+}
+
+func cleanupHpa(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
+	hpaList, err := client.AutoscalingV1().HorizontalPodAutoscalers(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
+	if err != nil {
+		return err
+	}
+
+	for _, hpa := range hpaList.Items {
+		id, ok := hpa.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
+		if ok && id != instanceId {
+			logger.Info("cleaning up HPA", zap.String("hpa", hpa.ObjectMeta.Name))
+			err := client.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Delete(hpa.ObjectMeta.Name, nil)
+			if err != nil {
+				logger.Error("error cleaning up HPA",
+					zap.Error(err),
+					zap.String("hpa_name", hpa.ObjectMeta.Name),
+					zap.String("hpa_namespace", hpa.ObjectMeta.Namespace))
+			}
+			// ignore err
+		}
+
+	}
+	return nil
+
+}
+
+// CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or
+// deletes the rolebindings completely if there are no Service Accounts in a rolebinding object.
+func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
+	for {
+		logger.Info("starting cleanupRoleBindings cycle")
+		// get all rolebindings ( just to be efficient, one call to kubernetes )
+		rbList, err := client.RbacV1beta1().RoleBindings(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
+		if err != nil {
+			// something wrong, but next iteration hopefully succeeds
+			logger.Error("error listing role bindings in all namespaces", zap.Error(err))
+			continue
+		}
+
+		// go through each role-binding object and do the cleanup necessary
+		for _, roleBinding := range rbList.Items {
+			// ignore role-bindings in kube-system namespace
+			if roleBinding.Namespace == "kube-system" {
+				continue
+			}
+
+			// ignore role-bindings not created by fission
+			if roleBinding.Name != types.PackageGetterRB && roleBinding.Name != types.SecretConfigMapGetterRB {
+				continue
+			}
+
+			// in order to find out if there are any functions that need this role-binding in role-binding namespace,
+			// we can list the functions once per role-binding.
+			funcList, err := fissionClient.Functions(roleBinding.Namespace).List(meta_v1.ListOptions{})
+			if err != nil {
+				logger.Error("error fetching environment list in namespace", zap.Error(err), zap.String("namespace", roleBinding.Namespace))
+				continue
+			}
+
+			// final map of service accounts that can be removed from this roleBinding object
+			// using a map here instead of a list so the code in RemoveSAFromRoleBindingWithRetries is efficient.
+			saToRemove := make(map[string]bool)
+
+			// the following flags are needed to decide if any of the service accounts can be removed from role-bindings depending on the functions that need them.
+			// ndmFunc denotes if there's at least one function that has executor type New deploy Manager
+			// funcEnvReference denotes if there's at least one function that has reference to an environment in the SA Namespace for the SA in question
+			var ndmFunc, funcEnvReference bool
+
+			// iterate through each subject in the role-binding and check if there are any references to them
+			for _, subj := range roleBinding.Subjects {
+				ndmFunc = false
+				funcEnvReference = false
+
+				// this is the reverse of what we're doing in setting up of role-bindings. if objects are created in default ns,
+				// the SA namespace will have the value of "fission-function"/"fission-builder" depending on the SA.
+				// so now we need to look for the objects in default namespace.
+				saNs := subj.Namespace
+				if subj.Namespace == functionNs ||
+					subj.Namespace == envBuilderNs {
+					saNs = meta_v1.NamespaceDefault
+				}
+
+				// go through each function and find out if there's either at least one function with env reference in the same namespace as the Service Account in this iteration
+				// or at least one function using ndm executor in the role-binding namespace and set the corresponding flags
+				for _, fn := range funcList.Items {
+					if fn.Spec.Environment.Namespace == saNs {
+						funcEnvReference = true
+						break
+					}
+
+					if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == types.ExecutorTypeNewdeploy {
+						ndmFunc = true
+						break
+					}
+				}
+
+				// if its a package-getterr-rb, we have 2 kinds of SAs and each of them is handled differently
+				// else if its a secret-configmap-rb, we have only one SA which is fission-fetcher
+				if roleBinding.Name == types.PackageGetterRB {
+					// check if there is an env obj in saNs
+					envList, err := fissionClient.Environments(saNs).List(meta_v1.ListOptions{})
+					if err != nil {
+						logger.Error("error fetching environment list in service account namespace", zap.Error(err), zap.String("namespace", saNs))
+						continue
+					}
+
+					// if the SA in this iteration is fission-builder, then we need to only check
+					// if either there's at least one env object in the SA's namespace, or,
+					// if there's at least one function in the role-binding namespace with env reference
+					// to the SA's namespace.
+					// if neither, then we can remove this SA from this role-binding
+					if subj.Name == types.FissionBuilderSA {
+						if len(envList.Items) == 0 && !funcEnvReference {
+							saToRemove[utils.MakeSAMapKey(subj.Name, subj.Namespace)] = true
+						}
+					}
+
+					// if the SA in this iteration is fission-fetcher, then in addition to above checks,
+					// we also need to check if there's at least one function with executor type New deploy
+					// in the rolebinding's namespace.
+					// if none of them are true, then remove this SA from this role-binding
+					if subj.Name == types.FissionFetcherSA {
+						if len(envList.Items) == 0 && !ndmFunc && !funcEnvReference {
+							// remove SA from rolebinding
+							saToRemove[utils.MakeSAMapKey(subj.Name, subj.Namespace)] = true
+						}
+					}
+				} else if roleBinding.Name == types.SecretConfigMapGetterRB {
+					// if there's not even one function in the role-binding's namespace and there's not even
+					// one function with env reference to the SA's namespace, then remove that SA
+					// from this role-binding
+					if !ndmFunc && !funcEnvReference {
+						saToRemove[utils.MakeSAMapKey(subj.Name, subj.Namespace)] = true
+					}
+				}
+			}
+
+			// finally, make a call to RemoveSAFromRoleBindingWithRetries for all the service accounts that need to be removed
+			// for the role-binding in this iteration
+			if len(saToRemove) != 0 {
+				logger.Debug("removing service accounts from role binding",
+					zap.Any("service_accounts", saToRemove),
+					zap.String("role_binding_name", roleBinding.Name),
+					zap.String("role_binding_namespace", roleBinding.Namespace))
+
+				// call this once in the end for each role-binding
+				err = utils.RemoveSAFromRoleBindingWithRetries(logger, client, roleBinding.Name, roleBinding.Namespace, saToRemove)
+				if err != nil {
+					// if there's an error, we just log it and proceed with the next role-binding, hoping that this role-binding
+					// will be processed in next iteration.
+					logger.Debug("error removing service account from role binding",
+						zap.Error(err),
+						zap.Any("service_accounts", saToRemove),
+						zap.String("role_binding_name", roleBinding.Name),
+						zap.String("role_binding_namespace", roleBinding.Namespace))
+				}
+			}
+		}
+
+		// some sleep before the next reaper iteration
+		time.Sleep(cleanupRoleBindingInterval)
+	}
+}

pkg/executor/util/merge.go

@@ -0,0 +1,219 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"errors"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/imdario/mergo"
+	apiv1 "k8s.io/api/core/v1"
+)
+
+// MergeContainerSpecs merges container specs using a predefined order.
+//
+// The order of the arguments indicates which spec has precedence (lower index takes precedence over higher indexes).
+// Slices and maps are merged; other fields are set only if they are a zero value.
+func MergeContainerSpecs(specs ...*apiv1.Container) apiv1.Container {
+	result := &apiv1.Container{}
+	for _, spec := range specs {
+		if spec == nil {
+			continue
+		}
+
+		err := mergo.Merge(result, spec)
+		if err != nil {
+			panic(err)
+		}
+	}
+	return *result
+}
+
+// mergeContainer is a specialized implementation of MergeContainerSpecs
+func mergeContainer(deployContainer *apiv1.Container, containerSpec apiv1.Container) error {
+
+	if &containerSpec == nil {
+		return nil
+	}
+
+	if deployContainer.Name == containerSpec.Name {
+		volMap := make(map[string]*apiv1.VolumeMount)
+		for _, vol := range deployContainer.VolumeMounts {
+			volMap[vol.Name] = &vol
+		}
+		for _, specVol := range containerSpec.VolumeMounts {
+			_, ok := volMap[specVol.Name]
+			if ok {
+				return errors.New("Duplicate volume name found in the spec")
+			} else {
+				deployContainer.VolumeMounts = append(deployContainer.VolumeMounts, specVol)
+			}
+		}
+		deployContainer.Env = append(deployContainer.Env, containerSpec.Env...)
+	}
+	return nil
+}
+
+func MergePodSpec(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
+	if &targetPodSpec == nil {
+		return nil
+	}
+
+	var multierr *multierror.Error
+
+	// Get item from spec, if they exist in deployment - merge, else append
+	// Same pattern for all lists (Mergo can not handle lists)
+	// At some point this is better done with generics/reflection?
+	err := mergeContainerLists(srcPodSpec, targetPodSpec)
+	if err != nil {
+		return err
+	}
+
+	err = mergeInitContainerList(srcPodSpec, targetPodSpec)
+	if err != nil {
+		return err
+	}
+
+	// For volumes - if duplicate exist, throw error
+	err = mergeVolumeLists(srcPodSpec, targetPodSpec)
+	if err != nil {
+		return err
+	}
+
+	if targetPodSpec.NodeName != "" {
+		srcPodSpec.NodeName = targetPodSpec.NodeName
+	}
+
+	if targetPodSpec.Subdomain != "" {
+		srcPodSpec.Subdomain = targetPodSpec.Subdomain
+	}
+
+	if targetPodSpec.SchedulerName != "" {
+		srcPodSpec.SchedulerName = targetPodSpec.SchedulerName
+	}
+
+	if targetPodSpec.PriorityClassName != "" {
+		srcPodSpec.PriorityClassName = targetPodSpec.PriorityClassName
+	}
+
+	if targetPodSpec.TerminationGracePeriodSeconds != nil {
+		srcPodSpec.TerminationGracePeriodSeconds = targetPodSpec.TerminationGracePeriodSeconds
+	}
+
+	//TODO - Security context should be merged instead of overriding.
+	if targetPodSpec.SecurityContext != nil {
+		srcPodSpec.SecurityContext = targetPodSpec.SecurityContext
+	}
+
+	//TODO - Affinity should be merged instead of overriding.
+	if targetPodSpec.Affinity != nil {
+		srcPodSpec.Affinity = targetPodSpec.Affinity
+	}
+
+	if targetPodSpec.Hostname != "" {
+		srcPodSpec.Hostname = targetPodSpec.Hostname
+	}
+
+	for _, obj := range targetPodSpec.ImagePullSecrets {
+		srcPodSpec.ImagePullSecrets = append(srcPodSpec.ImagePullSecrets, obj)
+	}
+
+	for _, obj := range targetPodSpec.Tolerations {
+		srcPodSpec.Tolerations = append(srcPodSpec.Tolerations, obj)
+	}
+
+	for _, obj := range targetPodSpec.HostAliases {
+		srcPodSpec.HostAliases = append(srcPodSpec.HostAliases, obj)
+	}
+
+	err = mergo.Merge(&srcPodSpec.NodeSelector, targetPodSpec.NodeSelector)
+	if err != nil {
+		multierr = multierror.Append(multierr, err)
+	}
+
+	return multierr.ErrorOrNil()
+}
+
+func mergeContainerLists(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
+	targetSpecContainers := targetPodSpec.Containers
+	targetContainers := make(map[string]apiv1.Container)
+	for _, c := range targetSpecContainers {
+		targetContainers[c.Name] = c
+	}
+
+	var multierr *multierror.Error
+	for _, c := range srcPodSpec.Containers {
+		container, ok := targetContainers[c.Name]
+		if ok {
+			err := mergeContainer(&c, container)
+			multierr = multierror.Append(multierr, err)
+			delete(targetContainers, c.Name)
+		}
+	}
+
+	for _, container := range targetContainers {
+		srcPodSpec.Containers = append(srcPodSpec.Containers, container)
+	}
+
+	return multierr.ErrorOrNil()
+}
+
+func mergeInitContainerList(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
+	targetSpecContainers := targetPodSpec.InitContainers
+	targetContainers := make(map[string]apiv1.Container)
+	for _, c := range targetSpecContainers {
+		targetContainers[c.Name] = c
+	}
+
+	var multierr *multierror.Error
+	for _, c := range srcPodSpec.InitContainers {
+		container, ok := targetContainers[c.Name]
+		if ok {
+			err := mergeContainer(&c, container)
+			multierr = multierror.Append(multierr, err)
+			delete(targetContainers, c.Name)
+		}
+	}
+
+	for _, container := range targetContainers {
+		srcPodSpec.InitContainers = append(srcPodSpec.InitContainers, container)
+	}
+	return multierr.ErrorOrNil()
+}
+
+func mergeVolumeLists(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
+	volumeList := targetPodSpec.Volumes
+	specVolumes := make(map[string]apiv1.Volume)
+	for _, vol := range volumeList {
+		specVolumes[vol.Name] = vol
+	}
+
+	var multierr *multierror.Error
+	for _, vol := range srcPodSpec.Volumes {
+		_, ok := specVolumes[vol.Name]
+		if ok {
+			multierr = multierror.Append(multierr, errors.New("Duplicate volume name found in the spec"))
+		} else {
+			delete(specVolumes, vol.Name)
+		}
+	}
+
+	for _, volume := range specVolumes {
+		srcPodSpec.Volumes = append(srcPodSpec.Volumes, volume)
+	}
+	return multierr.ErrorOrNil()
+}

pkg/executor/util/merge_test.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	apiv1 "k8s.io/api/core/v1"
+)
+
+func TestMergeContainerSpecs(t *testing.T) {
+	expected := apiv1.Container{
+		Name:  "containerName",
+		Image: "testImage",
+		Command: []string{
+			"command",
+		},
+		Args: []string{
+			"arg1",
+			"arg2",
+		},
+		ImagePullPolicy: apiv1.PullNever,
+		TTY:             true,
+		Env: []apiv1.EnvVar{
+			{
+				Name:  "a",
+				Value: "b",
+			},
+			{
+				Name:  "c",
+				Value: "d",
+			},
+		},
+	}
+	specs := []*apiv1.Container{
+		{
+			Name:  "containerName",
+			Image: "testImage",
+			Command: []string{
+				"command",
+			},
+			Args: []string{
+				"arg1",
+				"arg2",
+			},
+			ImagePullPolicy: apiv1.PullNever,
+			TTY:             true,
+		},
+		{
+			Name:  "shouldNotBeThere",
+			Image: "shouldNotBeThere",
+			Env: []apiv1.EnvVar{
+				{
+					Name:  "a",
+					Value: "b",
+				},
+			},
+			ImagePullPolicy: apiv1.PullAlways,
+			TTY:             false,
+		},
+		{
+			Env: []apiv1.EnvVar{
+				{
+					Name:  "c",
+					Value: "d",
+				},
+			},
+			ImagePullPolicy: apiv1.PullIfNotPresent,
+			TTY:             false,
+		},
+	}
+	result := MergeContainerSpecs(specs...)
+	assert.Equal(t, expected, result)
+
+	// Check if merging order actually matters
+	var rspecs []*apiv1.Container
+	for i := len(specs) - 1; i >= 0; i -= 1 {
+		rspecs = append(rspecs, specs[i])
+	}
+	reverseResult := MergeContainerSpecs(rspecs...)
+	assert.NotEqual(t, expected, reverseResult)
+}
+
+func TestMergeContainerSpecsSingle(t *testing.T) {
+	expected := apiv1.Container{
+		Name:  "containerName",
+		Image: "testImage",
+		Command: []string{
+			"command",
+		},
+		Args: []string{
+			"arg1",
+			"arg2",
+		},
+		ImagePullPolicy: apiv1.PullNever,
+		TTY:             true,
+	}
+	result := MergeContainerSpecs(&expected)
+	assert.EqualValues(t, expected, result)
+}
+
+func TestMergeContainerSpecsNil(t *testing.T) {
+	expected := apiv1.Container{}
+	result := MergeContainerSpecs()
+	assert.EqualValues(t, expected, result)
+}

pkg/featureconfig/config.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package featureconfig
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/ghodss/yaml"
+)
+
+// GetFeatureConfig reads the configMap file and unmarshals the config into a feature config struct
+func GetFeatureConfig() (*FeatureConfig, error) {
+	// read the file
+	b64EncodedContent, err := ioutil.ReadFile(FeatureConfigFile)
+	if err != nil {
+		return nil, fmt.Errorf("error reading YAML file %s: %v", FeatureConfigFile, err)
+	}
+
+	// b64 decode file
+	yamlContent, err := base64.StdEncoding.DecodeString(string(b64EncodedContent))
+	if err != nil {
+		return nil, fmt.Errorf("error b64 decoding the config : %v", err)
+	}
+
+	// unmarshal into feature config
+	featureConfig := &FeatureConfig{}
+	err = yaml.Unmarshal(yamlContent, featureConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshalling YAML config %v", err)
+	}
+
+	return featureConfig, err
+}

pkg/featureconfig/types.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package featureconfig
+
+const (
+	FeatureConfigFile = "/etc/config/config.yaml"
+	CanaryFeature     = "canary"
+)
+
+type (
+	// config.yaml contains config parameters for optional features
+	// To add new features with config parameters:
+	// 1. create a yaml block with feature name in charts/_helpers.tpl
+	// 2. define a corresponding struct with the feature config for the yaml unmarshal below
+	// 3. start the appropriate controllers needed for this feature
+
+	FeatureConfig struct {
+		// In the future more such feature configs can be added here for each optional feature
+		CanaryConfig CanaryFeatureConfig `json:"canary"`
+	}
+
+	// specific feature config
+	CanaryFeatureConfig struct {
+		IsEnabled     bool   `json:"enabled"`
+		PrometheusSvc string `json:"prometheusSvc"`
+	}
+)

pkg/fetcher/client/client.go

@@ -0,0 +1,106 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io/ioutil"
+
+	"go.uber.org/zap"
+
+	"net/http"
+	"strings"
+	"time"
+
+	"go.opencensus.io/plugin/ochttp"
+	"golang.org/x/net/context/ctxhttp"
+
+	ferror "github.com/fission/fission/pkg/error"
+	"github.com/fission/fission/pkg/types"
+)
+
+type (
+	Client struct {
+		logger     *zap.Logger
+		url        string
+		httpClient *http.Client
+	}
+)
+
+func MakeClient(logger *zap.Logger, fetcherUrl string) *Client {
+	return &Client{
+		logger: logger.Named("fetcher_client"),
+		url:    strings.TrimSuffix(fetcherUrl, "/"),
+		httpClient: &http.Client{
+			Transport: &ochttp.Transport{},
+		},
+	}
+}
+
+func (c *Client) getSpecializeUrl() string {
+	return c.url + "/specialize"
+}
+
+func (c *Client) getFetchUrl() string {
+	return c.url + "/fetch"
+}
+
+func (c *Client) getUploadUrl() string {
+	return c.url + "/upload"
+}
+
+func (c *Client) Specialize(ctx context.Context, req *types.FunctionSpecializeRequest) error {
+	_, err := sendRequest(c.logger, ctx, c.httpClient, req, c.getSpecializeUrl())
+	return err
+}
+
+func (c *Client) Fetch(ctx context.Context, fr *types.FunctionFetchRequest) error {
+	_, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getFetchUrl())
+	return err
+}
+
+func (c *Client) Upload(ctx context.Context, fr *types.ArchiveUploadRequest) (*types.ArchiveUploadResponse, error) {
+	body, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getUploadUrl())
+
+	uploadResp := types.ArchiveUploadResponse{}
+	err = json.Unmarshal(body, &uploadResp)
+	if err != nil {
+		return nil, err
+	}
+
+	return &uploadResp, nil
+}
+
+func sendRequest(logger *zap.Logger, ctx context.Context, httpClient *http.Client, req interface{}, url string) ([]byte, error) {
+	body, err := json.Marshal(req)
+	if err != nil {
+		return nil, err
+	}
+
+	maxRetries := 20
+	var resp *http.Response
+
+	for i := 0; i < maxRetries; i++ {
+		resp, err = ctxhttp.Post(ctx, httpClient, url, "application/json", bytes.NewReader(body))
+
+		if err == nil {
+			if resp.StatusCode == 200 {
+				body, err := ioutil.ReadAll(resp.Body)
+				if err != nil {
+					logger.Error("error reading response body", zap.Error(err))
+				}
+				resp.Body.Close()
+				return body, err
+			}
+			err = ferror.MakeErrorFromHTTP(resp)
+		}
+
+		if i < maxRetries-1 {
+			time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
+			logger.Error("error specializing/fetching/uploading package, retrying", zap.Error(err), zap.String("url", url))
+			continue
+		}
+	}
+
+	return nil, err
+}

pkg/fetcher/config/config.go

@@ -0,0 +1,304 @@
+package container
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+type Config struct {
+	fetcherImage           string
+	fetcherImagePullPolicy apiv1.PullPolicy
+
+	resourceRequirements apiv1.ResourceRequirements
+
+	// used by generic pool when creating env deployment to specify the share volume path for fetcher & env
+	// change this may break v1 compatibility, since most of the v1 environments have hard-coded "/userfunc" in loading path
+	sharedMountPath  string
+	sharedSecretPath string
+	sharedCfgMapPath string
+
+	dockerRegistryAuthDomain string
+	dockerRegistryUsername   string
+	dockerRegistryPassword   string
+
+	serviceAccount string
+
+	jaegerCollectorEndpoint string
+}
+
+func getFetcherResources() (apiv1.ResourceRequirements, error) {
+	mincpu, err := resource.ParseQuantity(os.Getenv("FETCHER_MINCPU"))
+	if err != nil {
+		return apiv1.ResourceRequirements{}, err
+	}
+
+	minmem, err := resource.ParseQuantity(os.Getenv("FETCHER_MINMEM"))
+	if err != nil {
+		return apiv1.ResourceRequirements{}, err
+	}
+
+	maxcpu, err := resource.ParseQuantity(os.Getenv("FETCHER_MAXCPU"))
+	if err != nil {
+		return apiv1.ResourceRequirements{}, err
+	}
+
+	maxmem, err := resource.ParseQuantity(os.Getenv("FETCHER_MAXMEM"))
+	if err != nil {
+		return apiv1.ResourceRequirements{}, err
+	}
+
+	return apiv1.ResourceRequirements{
+		Requests: map[apiv1.ResourceName]resource.Quantity{
+			apiv1.ResourceCPU:    mincpu,
+			apiv1.ResourceMemory: minmem,
+		},
+		Limits: map[apiv1.ResourceName]resource.Quantity{
+			apiv1.ResourceCPU:    maxcpu,
+			apiv1.ResourceMemory: maxmem,
+		},
+	}, nil
+}
+
+func MakeFetcherConfig(sharedMountPath string) (*Config, error) {
+	resources, err := getFetcherResources()
+	if err != nil {
+		return nil, err
+	}
+
+	fetcherImage := os.Getenv("FETCHER_IMAGE")
+	if len(fetcherImage) == 0 {
+		fetcherImage = "fission/fetcher"
+	}
+
+	fetcherImagePullPolicy := os.Getenv("FETCHER_IMAGE_PULL_POLICY")
+	if len(fetcherImagePullPolicy) == 0 {
+		fetcherImagePullPolicy = "IfNotPresent"
+	}
+
+	return &Config{
+		resourceRequirements:    resources,
+		fetcherImage:            fetcherImage,
+		fetcherImagePullPolicy:  utils.GetImagePullPolicy(fetcherImagePullPolicy),
+		sharedMountPath:         sharedMountPath,
+		sharedSecretPath:        "/secrets",
+		sharedCfgMapPath:        "/configs",
+		jaegerCollectorEndpoint: os.Getenv("OPENCENSUS_TRACE_JAEGER_COLLECTOR_ENDPOINT"),
+		serviceAccount:          types.FissionFetcherSA,
+	}, nil
+}
+
+func (cfg *Config) SetupServiceAccount(kubernetesClient *kubernetes.Clientset, namespace string, context interface{}) error {
+	_, err := utils.SetupSA(kubernetesClient, types.FissionFetcherSA, namespace)
+	if err != nil {
+		log.Printf("Error : %v creating %s in ns : %s for: %#v", err, types.FissionFetcherSA, namespace, context)
+		return err
+	}
+
+	return nil
+}
+
+func (cfg *Config) SharedMountPath() string {
+	return cfg.sharedMountPath
+}
+
+func (cfg *Config) NewSpecializeRequest(fn *fv1.Function, env *fv1.Environment) types.FunctionSpecializeRequest {
+	// for backward compatibility, since most v1 env
+	// still try to load user function from hard coded
+	// path /userfunc/user
+	targetFilename := "user"
+	if env.Spec.Version >= 2 {
+		targetFilename = string(fn.Metadata.UID)
+	}
+
+	return types.FunctionSpecializeRequest{
+		FetchReq: types.FunctionFetchRequest{
+			FetchType: types.FETCH_DEPLOYMENT,
+			Package: metav1.ObjectMeta{
+				Namespace: fn.Spec.Package.PackageRef.Namespace,
+				Name:      fn.Spec.Package.PackageRef.Name,
+			},
+			Filename:    targetFilename,
+			Secrets:     fn.Spec.Secrets,
+			ConfigMaps:  fn.Spec.ConfigMaps,
+			KeepArchive: env.Spec.KeepArchive,
+		},
+		LoadReq: types.FunctionLoadRequest{
+			FilePath:         filepath.Join(cfg.sharedMountPath, targetFilename),
+			FunctionName:     fn.Spec.Package.FunctionName,
+			FunctionMetadata: &fn.Metadata,
+			EnvVersion:       env.Spec.Version,
+		},
+	}
+}
+
+func (cfg *Config) AddFetcherToPodSpec(podSpec *apiv1.PodSpec, mainContainerName string) error {
+	return cfg.addFetcherToPodSpecWithCommand(podSpec, mainContainerName, cfg.fetcherCommand())
+}
+
+func (cfg *Config) AddSpecializingFetcherToPodSpec(podSpec *apiv1.PodSpec, mainContainerName string, fn *fv1.Function, env *fv1.Environment) error {
+	specializeReq := cfg.NewSpecializeRequest(fn, env)
+	specializePayload, err := json.Marshal(specializeReq)
+	if err != nil {
+		return err
+	}
+
+	return cfg.addFetcherToPodSpecWithCommand(
+		podSpec,
+		mainContainerName,
+		cfg.fetcherCommand(
+			"-specialize-on-startup",
+			"-specialize-request", string(specializePayload),
+		),
+	)
+}
+
+func (cfg *Config) fetcherCommand(extraArgs ...string) []string {
+	command := []string{"/fetcher",
+		"-secret-dir", cfg.sharedSecretPath,
+		"-cfgmap-dir", cfg.sharedCfgMapPath,
+		"-jaeger-collector-endpoint", cfg.jaegerCollectorEndpoint,
+	}
+
+	command = append(command, extraArgs...)
+	command = append(command, cfg.sharedMountPath)
+	return command
+}
+
+func (cfg *Config) volumesWithMounts() ([]apiv1.Volume, []apiv1.VolumeMount) {
+	volumes := []apiv1.Volume{
+		{
+			Name: types.SharedVolumeUserfunc,
+			VolumeSource: apiv1.VolumeSource{
+				EmptyDir: &apiv1.EmptyDirVolumeSource{},
+			},
+		},
+		{
+			Name: types.SharedVolumeSecrets,
+			VolumeSource: apiv1.VolumeSource{
+				EmptyDir: &apiv1.EmptyDirVolumeSource{},
+			},
+		},
+		{
+			Name: types.SharedVolumeConfigmaps,
+			VolumeSource: apiv1.VolumeSource{
+				EmptyDir: &apiv1.EmptyDirVolumeSource{},
+			},
+		},
+	}
+	mounts := []apiv1.VolumeMount{
+		{
+			Name:      types.SharedVolumeUserfunc,
+			MountPath: cfg.sharedMountPath,
+		},
+		{
+			Name:      types.SharedVolumeSecrets,
+			MountPath: cfg.sharedSecretPath,
+		},
+		{
+			Name:      types.SharedVolumeConfigmaps,
+			MountPath: cfg.sharedCfgMapPath,
+		},
+	}
+
+	return volumes, mounts
+}
+
+func (cfg *Config) addFetcherToPodSpecWithCommand(podSpec *apiv1.PodSpec, mainContainerName string, command []string) error {
+	volumes, mounts := cfg.volumesWithMounts()
+	c := apiv1.Container{
+		Name:                   "fetcher",
+		Command:                command,
+		Image:                  cfg.fetcherImage,
+		ImagePullPolicy:        cfg.fetcherImagePullPolicy,
+		TerminationMessagePath: "/dev/termination-log",
+		VolumeMounts:           mounts,
+		Resources:              cfg.resourceRequirements,
+		ReadinessProbe: &apiv1.Probe{
+			InitialDelaySeconds: 1,
+			PeriodSeconds:       1,
+			FailureThreshold:    30,
+			Handler: apiv1.Handler{
+				HTTPGet: &apiv1.HTTPGetAction{
+					Path: "/readniess-healthz",
+					Port: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 8000,
+					},
+				},
+			},
+		},
+		LivenessProbe: &apiv1.Probe{
+			InitialDelaySeconds: 1,
+			PeriodSeconds:       5,
+			Handler: apiv1.Handler{
+				HTTPGet: &apiv1.HTTPGetAction{
+					Path: "/healthz",
+					Port: intstr.IntOrString{
+						Type:   intstr.Int,
+						IntVal: 8000,
+					},
+				},
+			},
+		},
+	}
+
+	// Pod is removed from endpoints list for service when it's
+	// state became "Termination". We used preStop hook as the
+	// workaround for connection draining since pod maybe shutdown
+	// before grace period expires.
+	// https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods
+	// https://github.com/kubernetes/kubernetes/issues/47576#issuecomment-308900172
+	if podSpec.TerminationGracePeriodSeconds != nil {
+		c.Lifecycle = &apiv1.Lifecycle{
+			PreStop: &apiv1.Handler{
+				Exec: &apiv1.ExecAction{
+					Command: []string{
+						"/bin/sleep",
+						fmt.Sprintf("%v", *podSpec.TerminationGracePeriodSeconds),
+					},
+				},
+			},
+		}
+	}
+
+	found := false
+	for ix, container := range podSpec.Containers {
+		if container.Name != mainContainerName {
+			continue
+		}
+
+		found = true
+		container.VolumeMounts = append(container.VolumeMounts, mounts...)
+		podSpec.Containers[ix] = container
+	}
+	if !found {
+		existingContainerNames := make([]string, len(podSpec.Containers))
+		for _, existingContainer := range podSpec.Containers {
+			existingContainerNames = append(existingContainerNames, existingContainer.Name)
+		}
+		return fmt.Errorf("Could not find main container '%s' in given PodSpec. Found: %v",
+			mainContainerName,
+			existingContainerNames)
+	}
+
+	podSpec.Volumes = append(podSpec.Volumes, volumes...)
+	podSpec.Containers = append(podSpec.Containers, c)
+	if podSpec.ServiceAccountName == "" {
+		podSpec.ServiceAccountName = types.FissionFetcherSA
+	}
+
+	return nil
+}

pkg/fetcher/fetcher.go

@@ -0,0 +1,659 @@
+package fetcher
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/mholt/archiver"
+	"github.com/pkg/errors"
+	uuid "github.com/satori/go.uuid"
+	"go.opencensus.io/plugin/ochttp"
+	"go.uber.org/zap"
+	"golang.org/x/net/context/ctxhttp"
+	k8serr "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	ferror "github.com/fission/fission/pkg/error"
+	"github.com/fission/fission/pkg/info"
+	storageSvcClient "github.com/fission/fission/pkg/storagesvc/client"
+	"github.com/fission/fission/pkg/types"
+)
+
+type (
+	Fetcher struct {
+		logger           *zap.Logger
+		sharedVolumePath string
+		sharedSecretPath string
+		sharedConfigPath string
+		fissionClient    *crd.FissionClient
+		kubeClient       *kubernetes.Clientset
+		httpClient       *http.Client
+	}
+)
+
+func makeVolumeDir(dirPath string) error {
+	return os.MkdirAll(dirPath, os.ModeDir|0700)
+}
+
+func MakeFetcher(logger *zap.Logger, sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) {
+	fLogger := logger.Named("fetcher")
+	err := makeVolumeDir(sharedVolumePath)
+	if err != nil {
+		fLogger.Fatal("error creating shared volume directory", zap.Error(err), zap.String("directory", sharedVolumePath))
+	}
+	err = makeVolumeDir(sharedSecretPath)
+	if err != nil {
+		fLogger.Fatal("error creating shared secret directory", zap.Error(err), zap.String("directory", sharedSecretPath))
+	}
+	err = makeVolumeDir(sharedConfigPath)
+	if err != nil {
+		fLogger.Fatal("error creating shared config directory", zap.Error(err), zap.String("directory", sharedConfigPath))
+	}
+
+	fissionClient, kubeClient, _, err := crd.MakeFissionClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "error making the fission / kube client")
+	}
+	return &Fetcher{
+		logger:           fLogger,
+		sharedVolumePath: sharedVolumePath,
+		sharedSecretPath: sharedSecretPath,
+		sharedConfigPath: sharedConfigPath,
+		fissionClient:    fissionClient,
+		kubeClient:       kubeClient,
+		httpClient: &http.Client{
+			Transport: &ochttp.Transport{},
+		},
+	}, nil
+}
+
+func downloadUrl(ctx context.Context, httpClient *http.Client, url string, localPath string) error {
+	resp, err := ctxhttp.Get(ctx, httpClient, url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	w, err := os.Create(localPath)
+	if err != nil {
+		return err
+	}
+	defer w.Close()
+
+	_, err = io.Copy(w, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	// flushing write buffer to file
+	err = w.Sync()
+	if err != nil {
+		return err
+	}
+
+	err = os.Chmod(localPath, 0600)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getChecksum(path string) (*fv1.Checksum, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	hasher := sha256.New()
+	_, err = io.Copy(hasher, f)
+	if err != nil {
+		return nil, err
+	}
+
+	c := hex.EncodeToString(hasher.Sum(nil))
+
+	return &fv1.Checksum{
+		Type: fv1.ChecksumTypeSHA256,
+		Sum:  c,
+	}, nil
+}
+
+func verifyChecksum(fileChecksum, checksum *fv1.Checksum) error {
+	if checksum.Type != fv1.ChecksumTypeSHA256 {
+		return ferror.MakeError(ferror.ErrorInvalidArgument, "Unsupported checksum type")
+	}
+	if fileChecksum.Sum != checksum.Sum {
+		return ferror.MakeError(ferror.ErrorChecksumFail, "Checksum validation failed")
+	}
+	return nil
+}
+
+func writeSecretOrConfigMap(dataMap map[string][]byte, dirPath string) error {
+	for key, val := range dataMap {
+		writeFilePath := filepath.Join(dirPath, key)
+		err := ioutil.WriteFile(writeFilePath, val, 0600)
+		if err != nil {
+			return errors.Wrapf(err, "Failed to write file %s", writeFilePath)
+		}
+	}
+	return nil
+}
+
+func (fetcher *Fetcher) VersionHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	fmt.Fprintf(w, info.BuildInfo().String())
+}
+
+func (fetcher *Fetcher) FetchHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "only POST is supported on this endpoint", http.StatusMethodNotAllowed)
+		return
+	}
+
+	startTime := time.Now()
+	defer func() {
+		elapsed := time.Since(startTime)
+		fetcher.logger.Info("fetch request done", zap.Duration("elapsed_time", elapsed))
+	}()
+
+	// parse request
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		fetcher.logger.Error("error reading request body", zap.Error(err))
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	var req types.FunctionFetchRequest
+	err = json.Unmarshal(body, &req)
+	if err != nil {
+		fetcher.logger.Error("error parsing request body", zap.Error(err))
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	code, err := fetcher.Fetch(r.Context(), req)
+	if err != nil {
+		fetcher.logger.Error("error fetching", zap.Error(err))
+		http.Error(w, err.Error(), code)
+		return
+	}
+
+	fetcher.logger.Info("checking secrets/cfgmaps")
+	code, err = fetcher.FetchSecretsAndCfgMaps(req.Secrets, req.ConfigMaps)
+	if err != nil {
+		fetcher.logger.Error("error fetching secrets and config maps", zap.Error(err))
+		http.Error(w, err.Error(), code)
+		return
+	}
+
+	fetcher.logger.Info("completed fetch request")
+	// all done
+	w.WriteHeader(http.StatusOK)
+}
+
+func (fetcher *Fetcher) SpecializeHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, fmt.Sprintf("only POST is supported on this endpoint, %v received", r.Method), http.StatusMethodNotAllowed)
+		return
+	}
+
+	// parse request
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		fetcher.logger.Error("error reading request body", zap.Error(err))
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	var req types.FunctionSpecializeRequest
+	err = json.Unmarshal(body, &req)
+	if err != nil {
+		fetcher.logger.Error("error parsing request body", zap.Error(err))
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	err = fetcher.SpecializePod(r.Context(), req.FetchReq, req.LoadReq)
+	if err != nil {
+		fetcher.logger.Error("error specialing pod", zap.Error(err))
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	// all done
+	w.WriteHeader(http.StatusOK)
+}
+
+// Fetch takes FetchRequest and makes the fetch call
+// It returns the HTTP code and error if any
+func (fetcher *Fetcher) Fetch(ctx context.Context, req types.FunctionFetchRequest) (int, error) {
+	// check that the requested filename is not an empty string and error out if so
+	if len(req.Filename) == 0 {
+		e := "fetch request received for an empty file name"
+		fetcher.logger.Error(e, zap.Any("request", req))
+		return http.StatusBadRequest, errors.New(fmt.Sprintf("%s, request: %v", e, req))
+	}
+
+	// verify first if the file already exists.
+	if _, err := os.Stat(filepath.Join(fetcher.sharedVolumePath, req.Filename)); err == nil {
+		fetcher.logger.Info("requested file already exists at shared volume - skipping fetch",
+			zap.String("requested_file", req.Filename),
+			zap.String("shared_volume_path", fetcher.sharedVolumePath))
+		return http.StatusOK, nil
+	}
+
+	tmpFile := req.Filename + ".tmp"
+	tmpPath := filepath.Join(fetcher.sharedVolumePath, tmpFile)
+
+	if req.FetchType == types.FETCH_URL {
+		// fetch the file and save it to the tmp path
+		err := downloadUrl(ctx, fetcher.httpClient, req.Url, tmpPath)
+		if err != nil {
+			e := "failed to download url"
+			fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url))
+			return http.StatusBadRequest, errors.Wrapf(err, "%s: %s", e, req.Url)
+		}
+	} else {
+		// get pkg
+		pkg, err := fetcher.fissionClient.Packages(req.Package.Namespace).Get(req.Package.Name)
+		if err != nil {
+			e := "failed to get package"
+			fetcher.logger.Error(e,
+				zap.String("package_name", req.Package.Name),
+				zap.String("package_namespace", req.Package.Namespace))
+			return http.StatusInternalServerError, errors.Wrap(err, e)
+		}
+
+		var archive *fv1.Archive
+		if req.FetchType == types.FETCH_SOURCE {
+			archive = &pkg.Spec.Source
+		} else if req.FetchType == types.FETCH_DEPLOYMENT {
+			// sometimes, the user may invoke the function even before the source code is built into a deploy pkg.
+			// this results in executor sending a fetch request of type FETCH_DEPLOYMENT and since pkg.Spec.Deployment.Url will be empty,
+			// we hit this "Get : unsupported protocol scheme "" error.
+			// it may be useful to the user if we can send a more meaningful error in such a scenario.
+			if pkg.Status.BuildStatus != types.BuildStatusSucceeded && pkg.Status.BuildStatus != types.BuildStatusNone {
+				e := fmt.Sprintf("cannot fetch deployment: package build status was not %q", types.BuildStatusSucceeded)
+				fetcher.logger.Error(e,
+					zap.String("package_name", pkg.Metadata.Name),
+					zap.String("package_namespace", pkg.Metadata.Namespace),
+					zap.Any("package_build_status", pkg.Status.BuildStatus))
+				return http.StatusInternalServerError, errors.New(fmt.Sprintf("%s: pkg %s.%s has a status of %s", e, pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Status.BuildStatus))
+			}
+			archive = &pkg.Spec.Deployment
+		}
+		// get package data as literal or by url
+		if len(archive.Literal) > 0 {
+			// write pkg.Literal into tmpPath
+			err = ioutil.WriteFile(tmpPath, archive.Literal, 0600)
+			if err != nil {
+				e := "failed to write file"
+				fetcher.logger.Error(e, zap.Error(err), zap.String("location", tmpPath))
+				return http.StatusInternalServerError, errors.Wrapf(err, "%s %s", e, tmpPath)
+			}
+		} else {
+			// download and verify
+			err := downloadUrl(ctx, fetcher.httpClient, archive.URL, tmpPath)
+			if err != nil {
+				e := "failed to download url"
+				fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url))
+				return http.StatusBadRequest, errors.Wrapf(err, "%s %s", e, req.Url)
+			}
+
+			checksum, err := getChecksum(tmpPath)
+			if err != nil {
+				e := "failed to get checksum"
+				fetcher.logger.Error(e, zap.Error(err))
+				return http.StatusBadRequest, errors.Wrap(err, e)
+			}
+
+			err = verifyChecksum(checksum, &archive.Checksum)
+			if err != nil {
+				e := "failed to verify checksum"
+				fetcher.logger.Error(e, zap.Error(err))
+				return http.StatusBadRequest, errors.Wrap(err, e)
+			}
+		}
+	}
+
+	if archiver.Zip.Match(tmpPath) && !req.KeepArchive {
+		// unarchive tmp file to a tmp unarchive path
+		tmpUnarchivePath := filepath.Join(fetcher.sharedVolumePath, uuid.NewV4().String())
+		err := fetcher.unarchive(tmpPath, tmpUnarchivePath)
+		if err != nil {
+			fetcher.logger.Error("error unarchiving",
+				zap.Error(err),
+				zap.String("archive_location", tmpPath),
+				zap.String("target_location", tmpUnarchivePath))
+			return http.StatusInternalServerError, err
+		}
+
+		tmpPath = tmpUnarchivePath
+	}
+
+	// move tmp file to requested filename
+	renamePath := filepath.Join(fetcher.sharedVolumePath, req.Filename)
+	err := fetcher.rename(tmpPath, renamePath)
+	if err != nil {
+		fetcher.logger.Error("error renaming file",
+			zap.Error(err),
+			zap.String("original_path", tmpPath),
+			zap.String("rename_path", renamePath))
+		return http.StatusInternalServerError, err
+	}
+
+	fetcher.logger.Info("successfully placed", zap.String("location", renamePath))
+	return http.StatusOK, nil
+}
+
+// FetchSecretsAndCfgMaps fetches secrets and configmaps specified by user
+// It returns the HTTP code and error if any
+func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fv1.SecretReference, cfgmaps []fv1.ConfigMapReference) (int, error) {
+	if len(secrets) > 0 {
+		for _, secret := range secrets {
+			data, err := fetcher.kubeClient.CoreV1().Secrets(secret.Namespace).Get(secret.Name, metav1.GetOptions{})
+
+			if err != nil {
+				e := "error getting secret from kubeapi"
+
+				httpCode := http.StatusInternalServerError
+				if k8serr.IsNotFound(err) {
+					httpCode = http.StatusNotFound
+					e = "secret was not found in kubeapi"
+				}
+				fetcher.logger.Error(e,
+					zap.Error(err),
+					zap.String("secret_name", secret.Name),
+					zap.String("secret_namespace", secret.Namespace))
+
+				return httpCode, errors.New(e)
+			}
+
+			secretPath := filepath.Join(secret.Namespace, secret.Name)
+			secretDir := filepath.Join(fetcher.sharedSecretPath, secretPath)
+			err = os.MkdirAll(secretDir, os.ModeDir|0644)
+			if err != nil {
+				e := "failed to create directory for secret"
+				fetcher.logger.Error(e,
+					zap.Error(err),
+					zap.String("directory", secretDir),
+					zap.String("secret_name", secret.Name),
+					zap.String("secret_namespace", secret.Namespace))
+				return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, secretDir)
+			}
+			err = writeSecretOrConfigMap(data.Data, secretDir)
+			if err != nil {
+				fetcher.logger.Error("failed to write secret to file location",
+					zap.Error(err),
+					zap.String("location", secretDir),
+					zap.String("secret_name", secret.Name),
+					zap.String("secret_namespace", secret.Namespace))
+				return http.StatusInternalServerError, err
+			}
+		}
+	}
+
+	if len(cfgmaps) > 0 {
+		for _, config := range cfgmaps {
+			data, err := fetcher.kubeClient.CoreV1().ConfigMaps(config.Namespace).Get(config.Name, metav1.GetOptions{})
+
+			if err != nil {
+				e := "error getting configmap from kubeapi"
+
+				httpCode := http.StatusInternalServerError
+				if k8serr.IsNotFound(err) {
+					httpCode = http.StatusNotFound
+					e = "configmap was not found in kubeapi"
+				}
+				fetcher.logger.Error(e,
+					zap.Error(err),
+					zap.String("config_map_name", config.Name),
+					zap.String("config_map_namespace", config.Namespace))
+
+				return httpCode, errors.New(e)
+			}
+
+			configPath := filepath.Join(config.Namespace, config.Name)
+			configDir := filepath.Join(fetcher.sharedConfigPath, configPath)
+			err = os.MkdirAll(configDir, os.ModeDir|0644)
+			if err != nil {
+				e := "failed to create directory for configmap"
+				fetcher.logger.Error(e,
+					zap.Error(err),
+					zap.String("directory", configDir),
+					zap.String("config_map_name", config.Name),
+					zap.String("config_map_namespace", config.Namespace))
+				return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, configDir)
+			}
+			configMap := make(map[string][]byte)
+			for key, val := range data.Data {
+				configMap[key] = []byte(val)
+			}
+			err = writeSecretOrConfigMap(configMap, configDir)
+			if err != nil {
+				fetcher.logger.Error("failed to write configmap to file location",
+					zap.Error(err),
+					zap.String("location", configDir),
+					zap.String("config_map_name", config.Name),
+					zap.String("config_map_namespace", config.Namespace))
+				return http.StatusInternalServerError, err
+			}
+		}
+	}
+
+	return http.StatusOK, nil
+}
+
+func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		http.Error(w, "only POST is supported on this endpoint", http.StatusMethodNotAllowed)
+		return
+	}
+
+	startTime := time.Now()
+	defer func() {
+		elapsed := time.Since(startTime)
+		fetcher.logger.Info("upload request done", zap.Duration("elapsed_time", elapsed))
+	}()
+
+	// parse request
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		fetcher.logger.Error("error reading request body", zap.Error(err))
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	var req types.ArchiveUploadRequest
+	err = json.Unmarshal(body, &req)
+	if err != nil {
+		fetcher.logger.Error("error parsing request body", zap.Error(err))
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	fetcher.logger.Info("fetcher received upload request", zap.Any("request", req))
+
+	zipFilename := req.Filename + ".zip"
+	srcFilepath := filepath.Join(fetcher.sharedVolumePath, req.Filename)
+	dstFilepath := filepath.Join(fetcher.sharedVolumePath, zipFilename)
+
+	if req.ArchivePackage {
+		err = fetcher.archive(srcFilepath, dstFilepath)
+		if err != nil {
+			e := "error archiving zip file"
+			fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath))
+			http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
+			return
+		}
+	} else {
+		err = os.Rename(srcFilepath, dstFilepath)
+		if err != nil {
+			e := "error renaming the archive"
+			fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath))
+			http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
+			return
+		}
+	}
+
+	fetcher.logger.Info("starting upload...")
+	ssClient := storageSvcClient.MakeClient(req.StorageSvcUrl)
+
+	fileID, err := ssClient.Upload(r.Context(), dstFilepath, nil)
+	if err != nil {
+		e := "error uploading zip file"
+		fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath))
+		http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
+		return
+	}
+
+	sum, err := getChecksum(dstFilepath)
+	if err != nil {
+		e := "error calculating checksum of zip file"
+		fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath))
+		http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
+		return
+	}
+
+	resp := types.ArchiveUploadResponse{
+		ArchiveDownloadUrl: ssClient.GetUrl(fileID),
+		Checksum:           *sum,
+	}
+
+	rBody, err := json.Marshal(resp)
+	if err != nil {
+		e := "error encoding upload response"
+		fetcher.logger.Error(e, zap.Error(err))
+		http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
+		return
+	}
+
+	fetcher.logger.Info("completed upload request")
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	w.Write(rBody)
+}
+
+func (fetcher *Fetcher) rename(src string, dst string) error {
+	err := os.Rename(src, dst)
+	if err != nil {
+		return errors.Wrap(err, "failed to move file")
+	}
+	return nil
+}
+
+// archive zips the contents of directory at src into a new zip file
+// at dst (note that the contents are zipped, not the directory itself).
+func (fetcher *Fetcher) archive(src string, dst string) error {
+	var files []string
+	target, err := os.Stat(src)
+	if err != nil {
+		return errors.Wrap(err, "failed to zip file")
+	}
+	if target.IsDir() {
+		// list all
+		fs, _ := ioutil.ReadDir(src)
+		for _, f := range fs {
+			files = append(files, filepath.Join(src, f.Name()))
+		}
+	} else {
+		files = append(files, src)
+	}
+	return archiver.Zip.Make(dst, files)
+}
+
+// unarchive is a function that unzips a zip file to destination
+func (fetcher *Fetcher) unarchive(src string, dst string) error {
+	err := archiver.Zip.Open(src, dst)
+	if err != nil {
+		return errors.Wrap(err, "failed to unzip file")
+	}
+	return nil
+}
+
+func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq types.FunctionFetchRequest, loadReq types.FunctionLoadRequest) error {
+	startTime := time.Now()
+	defer func() {
+		elapsed := time.Since(startTime)
+		fetcher.logger.Info("specialize request done", zap.Duration("elapsed_time", elapsed))
+	}()
+
+	_, err := fetcher.Fetch(ctx, fetchReq)
+	if err != nil {
+		return errors.Wrap(err, "error fetching deploy package")
+	}
+
+	_, err = fetcher.FetchSecretsAndCfgMaps(fetchReq.Secrets, fetchReq.ConfigMaps)
+	if err != nil {
+		return errors.Wrap(err, "error fetching secrets/configs")
+	}
+
+	// Specialize the pod
+
+	maxRetries := 30
+	var contentType string
+	var specializeURL string
+	var reader *bytes.Reader
+
+	loadPayload, err := json.Marshal(loadReq)
+	if err != nil {
+		return errors.Wrap(err, "error encoding load request")
+	}
+
+	if loadReq.EnvVersion >= 2 {
+		contentType = "application/json"
+		specializeURL = "http://localhost:8888/v2/specialize"
+		reader = bytes.NewReader(loadPayload)
+		fetcher.logger.Info("calling environment v2 specialization endpoint")
+	} else {
+		contentType = "text/plain"
+		specializeURL = "http://localhost:8888/specialize"
+		reader = bytes.NewReader([]byte{})
+		fetcher.logger.Info("calling environment v1 specialization endpoint")
+	}
+
+	for i := 0; i < maxRetries; i++ {
+		resp, err := http.Post(specializeURL, contentType, reader)
+		if err == nil && resp.StatusCode < 300 {
+			// Success
+			resp.Body.Close()
+			return nil
+		}
+
+		// Only retry for the specific case of a connection error.
+		if urlErr, ok := err.(*url.Error); ok {
+			if netErr, ok := urlErr.Err.(*net.OpError); ok {
+				if netErr.Op == "dial" {
+					if i < maxRetries-1 {
+						time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
+						fetcher.logger.Error("error connecting to function environment pod for specialization request, retrying", zap.Error(netErr))
+						continue
+					}
+				}
+			}
+		}
+
+		if err == nil {
+			err = ferror.MakeErrorFromHTTP(resp)
+		}
+
+		return errors.Wrap(err, "error specializing function pod")
+	}
+
+	return errors.Wrapf(err, "error specializing function pod after %v times", maxRetries)
+}

pkg/fission-cli/buildwatch.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/controller/client"
+	"github.com/fission/fission/pkg/fission-cli/util"
+	"github.com/fission/fission/pkg/types"
+)
+
+type (
+	// packageBuildWatcher is used to watch a set of in-progress builds.
+	packageBuildWatcher struct {
+		// fission client
+		fclient *client.Client
+
+		// set of packages already printed, ensures we don't duplicate the notifications
+		finished map[string]bool
+
+		// set of metadata in the app spec.  packages outside this set should be ignored.
+		pkgMeta map[string]metav1.ObjectMeta
+	}
+)
+
+func makePackageBuildWatcher(fclient *client.Client) *packageBuildWatcher {
+	return &packageBuildWatcher{
+		fclient:  fclient,
+		finished: make(map[string]bool),
+		pkgMeta:  make(map[string]metav1.ObjectMeta),
+	}
+}
+
+func (w *packageBuildWatcher) addPackages(pkgMeta map[string]metav1.ObjectMeta) {
+	for k, v := range pkgMeta {
+		w.pkgMeta[k] = v
+	}
+}
+
+func (w *packageBuildWatcher) watch(ctx context.Context) {
+	for {
+		// non-blocking check if we're cancelled
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		// poll list of packages (TODO: convert to watch)
+		pkgs, err := w.fclient.PackageList(metav1.NamespaceAll)
+		util.CheckErr(err, "Getting list of packages")
+
+		// find packages that (a) are in the app spec and (b) have an interesting
+		// build status (either succeeded or failed; not "none")
+		keepWaiting := false
+		buildpkgs := make([]fv1.Package, 0)
+		for _, pkg := range pkgs {
+			_, ok := w.pkgMeta[mapKey(&pkg.Metadata)]
+			if !ok {
+				continue
+			}
+			if pkg.Status.BuildStatus == types.BuildStatusNone {
+				continue
+			}
+			if pkg.Status.BuildStatus == types.BuildStatusPending ||
+				pkg.Status.BuildStatus == types.BuildStatusRunning {
+				keepWaiting = true
+			}
+			buildpkgs = append(buildpkgs, pkg)
+		}
+
+		// print package status, and error logs if any
+		for _, pkg := range buildpkgs {
+			k := pkgKey(&pkg)
+			if _, printed := w.finished[k]; printed {
+				continue
+			}
+			if pkg.Status.BuildStatus == types.BuildStatusFailed {
+				w.finished[k] = true
+				fmt.Printf("--- Build FAILED: ---\n%v\n------\n", pkg.Status.BuildLog)
+			} else if pkg.Status.BuildStatus == types.BuildStatusSucceeded {
+				w.finished[k] = true
+				fmt.Printf("--- Build SUCCEEDED ---\n")
+				if len(pkg.Status.BuildLog) > 0 {
+					fmt.Printf("%v\n------\n", pkg.Status.BuildLog)
+				}
+			}
+		}
+
+		// if there are no builds running, we can stop polling
+		if !keepWaiting {
+			return
+		}
+		time.Sleep(time.Second)
+	}
+}
+
+func pkgKey(pkg *fv1.Package) string {
+	// packages are mutable so we want to keep track of them by resource version
+	return fmt.Sprintf("%v:%v:%v", pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Metadata.ResourceVersion)
+}

pkg/fission-cli/canaryconfig.go

@@ -0,0 +1,235 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/util"
+	"github.com/fission/fission/pkg/types"
+)
+
+func canaryConfigCreate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	canaryConfigName := c.String("name")
+	// canary configs can be created for functions in the same namespace
+	if len(canaryConfigName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+
+	trigger := c.String("httptrigger")
+	newFunc := c.String("newfunction")
+	oldFunc := c.String("oldfunction")
+	ns := c.String("fnNamespace")
+	incrementStep := c.Int("increment-step")
+	failureThreshold := c.Int("failure-threshold")
+	incrementInterval := c.String("increment-interval")
+
+	// check for time parsing
+	_, err := time.ParseDuration(incrementInterval)
+	util.CheckErr(err, "parsing time duration.")
+
+	// check that the trigger exists in the same namespace.
+	m := &metav1.ObjectMeta{
+		Name:      trigger,
+		Namespace: ns,
+	}
+
+	htTrigger, err := client.HTTPTriggerGet(m)
+	if err != nil {
+		util.CheckErr(err, "find trigger referenced in the canary config")
+	}
+
+	// check that the trigger has function reference type function weights
+	if htTrigger.Spec.FunctionReference.Type != types.FunctionReferenceTypeFunctionWeights {
+		log.Fatal("Canary config cannot be created for http triggers that do not reference functions by weights")
+	}
+
+	// check that the trigger references same functions in the function weights
+	_, ok := htTrigger.Spec.FunctionReference.FunctionWeights[newFunc]
+	if !ok {
+		log.Fatal(fmt.Sprintf("HTTP Trigger doesn't reference the function %s in Canary Config", newFunc))
+	}
+
+	_, ok = htTrigger.Spec.FunctionReference.FunctionWeights[oldFunc]
+	if !ok {
+		log.Fatal(fmt.Sprintf("HTTP Trigger doesn't reference the function %s in Canary Config", oldFunc))
+	}
+
+	// check that the functions exist in the same namespace
+	fnList := []string{newFunc, oldFunc}
+	err = util.CheckFunctionExistence(client, fnList, ns)
+	if err != nil {
+		log.Fatal(fmt.Sprintf("checkFunctionExistence err : %v", err))
+	}
+
+	// finally create canaryCfg in the same namespace as the functions referenced
+	canaryCfg := &fv1.CanaryConfig{
+		Metadata: metav1.ObjectMeta{
+			Name:      canaryConfigName,
+			Namespace: ns,
+		},
+		Spec: fv1.CanaryConfigSpec{
+			Trigger:                 trigger,
+			NewFunction:             newFunc,
+			OldFunction:             oldFunc,
+			WeightIncrement:         incrementStep,
+			WeightIncrementDuration: incrementInterval,
+			FailureThreshold:        failureThreshold,
+			FailureType:             fv1.FailureTypeStatusCode,
+		},
+		Status: fv1.CanaryConfigStatus{
+			Status: fv1.CanaryConfigStatusPending,
+		},
+	}
+
+	_, err = client.CanaryConfigCreate(canaryCfg)
+	util.CheckErr(err, "create canary config")
+
+	fmt.Printf("canary config '%v' created\n", canaryConfigName)
+	return err
+}
+
+func canaryConfigGet(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	name := c.String("name")
+	if len(name) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+	ns := c.String("canaryNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      name,
+		Namespace: ns,
+	}
+
+	canaryCfg, err := client.CanaryConfigGet(m)
+	util.CheckErr(err, "get canary config")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "TRIGGER", "FUNCTION-N", "FUNCTION-N-1", "WEIGHT-INCREMENT", "INTERVAL", "FAILURE-THRESHOLD", "FAILURE-TYPE", "STATUS")
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+		canaryCfg.Metadata.Name, canaryCfg.Spec.Trigger, canaryCfg.Spec.NewFunction, canaryCfg.Spec.OldFunction, canaryCfg.Spec.WeightIncrement, canaryCfg.Spec.WeightIncrementDuration,
+		canaryCfg.Spec.FailureThreshold, canaryCfg.Spec.FailureType, canaryCfg.Status.Status)
+
+	w.Flush()
+	return nil
+}
+
+func canaryConfigUpdate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	canaryConfigName := c.String("name")
+	ns := c.String("canaryNamespace")
+	if len(canaryConfigName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+
+	incrementStep := c.Int("increment-step")
+	failureThreshold := c.Int("failure-threshold")
+	incrementInterval := c.String("increment-interval")
+
+	// check for time parsing
+	_, err := time.ParseDuration(incrementInterval)
+	util.CheckErr(err, "parsing time duration.")
+
+	// get the current config
+	m := &metav1.ObjectMeta{
+		Name:      canaryConfigName,
+		Namespace: ns,
+	}
+
+	var updateNeeded bool
+	canaryCfg, err := client.CanaryConfigGet(m)
+	util.CheckErr(err, "get canary config")
+
+	if incrementStep != canaryCfg.Spec.WeightIncrement {
+		canaryCfg.Spec.WeightIncrement = incrementStep
+		updateNeeded = true
+	}
+
+	if failureThreshold != canaryCfg.Spec.FailureThreshold {
+		canaryCfg.Spec.FailureThreshold = failureThreshold
+		updateNeeded = true
+	}
+
+	if incrementInterval != canaryCfg.Spec.WeightIncrementDuration {
+		canaryCfg.Spec.WeightIncrementDuration = incrementInterval
+		updateNeeded = true
+	}
+
+	if updateNeeded {
+		canaryCfg.Status.Status = fv1.CanaryConfigStatusPending
+
+		_, err = client.CanaryConfigUpdate(canaryCfg)
+		util.CheckErr(err, "update canary config")
+	}
+
+	return nil
+}
+
+func canaryConfigDelete(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	canaryConfigName := c.String("name")
+	ns := c.String("canaryNamespace")
+	if len(canaryConfigName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+
+	// get the current config
+	m := &metav1.ObjectMeta{
+		Name:      canaryConfigName,
+		Namespace: ns,
+	}
+
+	err := client.CanaryConfigDelete(m)
+	util.CheckErr(err, fmt.Sprintf("delete function '%v.%v'", canaryConfigName, ns))
+
+	fmt.Printf("canaryconfig '%v.%v' deleted\n", canaryConfigName, ns)
+	return err
+}
+
+func canaryConfigList(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	ns := c.String("canaryNamespace")
+
+	canaryCfgs, err := client.CanaryConfigList(ns)
+	util.CheckErr(err, "list canary config")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "TRIGGER", "FUNCTION-N", "FUNCTION-N-1", "WEIGHT-INCREMENT", "INTERVAL", "FAILURE-THRESHOLD", "FAILURE-TYPE", "STATUS")
+	for _, canaryCfg := range canaryCfgs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+			canaryCfg.Metadata.Name, canaryCfg.Spec.Trigger, canaryCfg.Spec.NewFunction, canaryCfg.Spec.OldFunction, canaryCfg.Spec.WeightIncrement, canaryCfg.Spec.WeightIncrementDuration,
+			canaryCfg.Spec.FailureThreshold, canaryCfg.Spec.FailureType, canaryCfg.Status.Status)
+	}
+
+	w.Flush()
+	return nil
+}

pkg/fission-cli/environment.go

@@ -0,0 +1,353 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"text/tabwriter"
+
+	"github.com/urfave/cli"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/controller/client"
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/util"
+)
+
+func getFunctionsByEnvironment(client *client.Client, envName, envNamespace string) ([]fv1.Function, error) {
+	fnList, err := client.FunctionList(metav1.NamespaceAll)
+	if err != nil {
+		return nil, err
+	}
+	fns := []fv1.Function{}
+	for _, fn := range fnList {
+		if fn.Spec.Environment.Name == envName && fn.Spec.Environment.Namespace == envNamespace {
+			fns = append(fns, fn)
+		}
+	}
+	return fns, nil
+}
+
+func envCreate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	envName := c.String("name")
+	if len(envName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+	envNamespace := c.String("envNamespace")
+
+	envList, err := client.EnvironmentList(envNamespace)
+	if err == nil && len(envList) > 0 {
+		log.Verbose(2, "%d environment(s) are present in the %s namespace.  "+
+			"These environments are not isolated from each other; use separate namespaces if you need isolation.",
+			len(envList), envNamespace)
+	}
+
+	var poolsize int
+	if c.IsSet("poolsize") {
+		poolsize = c.Int("poolsize")
+	} else {
+		poolsize = 3
+	}
+
+	envImg := c.String("image")
+	if len(envImg) == 0 {
+		log.Fatal("Need an image, use --image.")
+	}
+
+	envVersion := c.Int("version")
+	envBuilderImg := c.String("builder")
+	envBuildCmd := c.String("buildcmd")
+	envExternalNetwork := c.Bool("externalnetwork")
+	envGracePeriod := c.Int64("period")
+	if envGracePeriod <= 0 {
+		envGracePeriod = 360
+	}
+
+	if len(envBuilderImg) > 0 {
+		if !c.IsSet("version") {
+			envVersion = 2
+		}
+		if len(envBuildCmd) == 0 {
+			envBuildCmd = "build"
+		}
+	}
+	if c.IsSet("poolsize") {
+		envVersion = 3
+	}
+
+	keepArchive := c.Bool("keeparchive")
+
+	// Environment API interface version is not specified and
+	// builder image is empty, set default interface version
+	if envVersion == 0 {
+		envVersion = 1
+	}
+
+	resourceReq := getResourceReq(c, v1.ResourceRequirements{})
+
+	env := &fv1.Environment{
+		Metadata: metav1.ObjectMeta{
+			Name:      envName,
+			Namespace: envNamespace,
+		},
+		Spec: fv1.EnvironmentSpec{
+			Version: envVersion,
+			Runtime: fv1.Runtime{
+				Image: envImg,
+			},
+			Builder: fv1.Builder{
+				Image:   envBuilderImg,
+				Command: envBuildCmd,
+			},
+			Poolsize:                     poolsize,
+			Resources:                    resourceReq,
+			AllowAccessToExternalNetwork: envExternalNetwork,
+			TerminationGracePeriod:       envGracePeriod,
+			KeepArchive:                  keepArchive,
+		},
+	}
+
+	// if we're writing a spec, don't call the API
+	if c.Bool("spec") {
+		specFile := fmt.Sprintf("env-%v.yaml", envName)
+		err := specSave(*env, specFile)
+		util.CheckErr(err, "create environment spec")
+		return nil
+	}
+
+	_, err = client.EnvironmentCreate(env)
+	util.CheckErr(err, "create environment")
+
+	fmt.Printf("environment '%v' created\n", envName)
+	return err
+}
+
+func envGet(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	envName := c.String("name")
+	if len(envName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+	envNamespace := c.String("envNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      envName,
+		Namespace: envNamespace,
+	}
+	env, err := client.EnvironmentGet(m)
+	util.CheckErr(err, "get environment")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE")
+	fmt.Fprintf(w, "%v\t%v\t%v\n",
+		env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image)
+	w.Flush()
+	return nil
+}
+
+func envUpdate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	envName := c.String("name")
+	if len(envName) == 0 {
+		log.Fatal("Need a name, use --name.")
+	}
+	envNamespace := c.String("envNamespace")
+
+	envImg := c.String("image")
+	envBuilderImg := c.String("builder")
+	envBuildCmd := c.String("buildcmd")
+	envExternalNetwork := c.Bool("externalnetwork")
+
+	if len(envImg) == 0 && len(envBuilderImg) == 0 && len(envBuildCmd) == 0 {
+		log.Fatal("Need --image to specify env image, or use --builder to specify env builder, or use --buildcmd to specify new build command.")
+	}
+
+	env, err := client.EnvironmentGet(&metav1.ObjectMeta{
+		Name:      envName,
+		Namespace: envNamespace,
+	})
+	util.CheckErr(err, "find environment")
+
+	if len(envImg) > 0 {
+		env.Spec.Runtime.Image = envImg
+	}
+
+	if env.Spec.Version == 1 && (len(envBuilderImg) > 0 || len(envBuildCmd) > 0) {
+		log.Fatal("Version 1 Environments do not support builders. Must specify --version=2.")
+	}
+
+	if len(envBuilderImg) > 0 {
+		env.Spec.Builder.Image = envBuilderImg
+	}
+	if len(envBuildCmd) > 0 {
+		env.Spec.Builder.Command = envBuildCmd
+	}
+
+	if c.IsSet("poolsize") {
+		env.Spec.Poolsize = c.Int("poolsize")
+	}
+
+	if c.IsSet("period") {
+		env.Spec.TerminationGracePeriod = c.Int64("period")
+	}
+
+	if c.IsSet("keeparchive") {
+		env.Spec.KeepArchive = c.Bool("keeparchive")
+	}
+
+	env.Spec.AllowAccessToExternalNetwork = envExternalNetwork
+
+	if c.IsSet("mincpu") || c.IsSet("maxcpu") || c.IsSet("minmemory") || c.IsSet("maxmemory") || c.IsSet("minscale") || c.IsSet("maxscale") {
+		log.Fatal("Updating resource limits/requests for existing environments is currently unsupported; re-create the environment instead.")
+	}
+
+	_, err = client.EnvironmentUpdate(env)
+	util.CheckErr(err, "update environment")
+
+	fmt.Printf("environment '%v' updated\n", envName)
+	return nil
+}
+
+func envDelete(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	envName := c.String("name")
+	if len(envName) == 0 {
+		log.Fatal("Need a name , use --name.")
+	}
+	envNamespace := c.String("envNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      envName,
+		Namespace: envNamespace,
+	}
+	err := client.EnvironmentDelete(m)
+	util.CheckErr(err, "delete environment")
+
+	fmt.Printf("environment '%v' deleted\n", envName)
+	return nil
+}
+
+func envList(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+	envNamespace := c.String("envNamespace")
+
+	envs, err := client.EnvironmentList(envNamespace)
+	util.CheckErr(err, "list environments")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "IMAGE", "BUILDER_IMAGE", "POOLSIZE", "MINCPU", "MAXCPU", "MINMEMORY", "MAXMEMORY", "EXTNET", "GRACETIME")
+	for _, env := range envs {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+			env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image, env.Spec.Builder.Image, env.Spec.Poolsize,
+			env.Spec.Resources.Requests.Cpu(), env.Spec.Resources.Limits.Cpu(),
+			env.Spec.Resources.Requests.Memory(), env.Spec.Resources.Limits.Memory(),
+			env.Spec.AllowAccessToExternalNetwork, env.Spec.TerminationGracePeriod)
+	}
+	w.Flush()
+
+	return nil
+}
+
+func getResourceReq(c *cli.Context, resources v1.ResourceRequirements) v1.ResourceRequirements {
+
+	var requestResources map[v1.ResourceName]resource.Quantity
+
+	if len(resources.Requests) == 0 {
+		requestResources = make(map[v1.ResourceName]resource.Quantity)
+	} else {
+		requestResources = resources.Requests
+	}
+
+	if c.IsSet("mincpu") {
+		mincpu := c.Int("mincpu")
+		cpuRequest, err := resource.ParseQuantity(strconv.Itoa(mincpu) + "m")
+		if err != nil {
+			log.Fatal("Failed to parse mincpu")
+		}
+		requestResources[v1.ResourceCPU] = cpuRequest
+	}
+
+	if c.IsSet("minmemory") {
+		minmem := c.Int("minmemory")
+		memRequest, err := resource.ParseQuantity(strconv.Itoa(minmem) + "Mi")
+		if err != nil {
+			log.Fatal("Failed to parse minmemory")
+		}
+		requestResources[v1.ResourceMemory] = memRequest
+	}
+
+	var limitResources map[v1.ResourceName]resource.Quantity
+
+	if len(resources.Limits) == 0 {
+		limitResources = make(map[v1.ResourceName]resource.Quantity)
+	} else {
+		limitResources = resources.Limits
+	}
+
+	if c.IsSet("maxcpu") {
+		maxcpu := c.Int("maxcpu")
+		cpuLimit, err := resource.ParseQuantity(strconv.Itoa(maxcpu) + "m")
+		if err != nil {
+			log.Fatal("Failed to parse maxcpu")
+		}
+		limitResources[v1.ResourceCPU] = cpuLimit
+	}
+
+	if c.IsSet("maxmemory") {
+		maxmem := c.Int("maxmemory")
+		memLimit, err := resource.ParseQuantity(strconv.Itoa(maxmem) + "Mi")
+		if err != nil {
+			log.Fatal("Failed to parse maxmemory")
+		}
+		limitResources[v1.ResourceMemory] = memLimit
+	}
+
+	limitCPU := limitResources[v1.ResourceCPU]
+	requestCPU := requestResources[v1.ResourceCPU]
+
+	if limitCPU.IsZero() && !requestCPU.IsZero() {
+		limitResources[v1.ResourceCPU] = requestCPU
+	} else if limitCPU.Cmp(requestCPU) < 0 {
+		log.Fatal(fmt.Sprintf("MinCPU (%v) cannot be greater than MaxCPU (%v)", requestCPU.String(), limitCPU.String()))
+	}
+
+	limitMem := limitResources[v1.ResourceMemory]
+	requestMem := requestResources[v1.ResourceMemory]
+
+	if limitMem.IsZero() && !requestMem.IsZero() {
+		limitResources[v1.ResourceMemory] = requestMem
+	} else if limitMem.Cmp(requestMem) < 0 {
+		log.Fatal(fmt.Sprintf("MinMemory (%v) cannot be greater than MaxMemory (%v)", requestMem.String(), limitMem.String()))
+	}
+
+	resources = v1.ResourceRequirements{
+		Requests: requestResources,
+		Limits:   limitResources,
+	}
+
+	return resources
+}

pkg/fission-cli/function.go

@@ -0,0 +1,851 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/fission/fission/pkg/types"
+	"github.com/satori/go.uuid"
+	"github.com/urfave/cli"
+	apiv1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	ferror "github.com/fission/fission/pkg/error"
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/logdb"
+	"github.com/fission/fission/pkg/fission-cli/util"
+)
+
+const (
+	DEFAULT_MIN_SCALE             = 1
+	DEFAULT_TARGET_CPU_PERCENTAGE = 80
+)
+
+func printPodLogs(c *cli.Context) error {
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		log.Fatal("Need --name argument.")
+	}
+
+	queryURL, err := url.Parse(util.GetServerUrl())
+	util.CheckErr(err, "parse the base URL")
+	queryURL.Path = fmt.Sprintf("/proxy/logs/%s", fnName)
+
+	req, err := http.NewRequest("POST", queryURL.String(), nil)
+	util.CheckErr(err, "create logs request")
+
+	httpClient := http.Client{}
+	resp, err := httpClient.Do(req)
+	util.CheckErr(err, "execute get logs request")
+
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return errors.New("get logs from pod directly")
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	util.CheckErr(err, "read the response body")
+	fmt.Println(string(body))
+	return nil
+}
+
+func getInvokeStrategy(c *cli.Context, existingInvokeStrategy *fv1.InvokeStrategy) (strategy *fv1.InvokeStrategy, err error) {
+
+	var fnExecutor, newFnExecutor fv1.ExecutorType
+
+	switch c.String("executortype") {
+	case "":
+		fallthrough
+	case types.ExecutorTypePoolmgr:
+		newFnExecutor = types.ExecutorTypePoolmgr
+	case types.ExecutorTypeNewdeploy:
+		newFnExecutor = types.ExecutorTypeNewdeploy
+	default:
+		return nil, errors.New("Executor type must be one of 'poolmgr' or 'newdeploy', defaults to 'poolmgr'")
+	}
+
+	if existingInvokeStrategy != nil {
+		fnExecutor = existingInvokeStrategy.ExecutionStrategy.ExecutorType
+
+		// override the executor type if user specified a new executor type
+		if c.IsSet("executortype") {
+			fnExecutor = newFnExecutor
+		}
+	} else {
+		fnExecutor = newFnExecutor
+	}
+
+	if fnExecutor == types.ExecutorTypePoolmgr {
+		if c.IsSet("targetcpu") || c.IsSet("minscale") || c.IsSet("maxscale") {
+			log.Fatal("To set target CPU or min/max scale for function, please specify \"--executortype newdeploy\"")
+		}
+
+		if c.IsSet("mincpu") || c.IsSet("maxcpu") || c.IsSet("minmemory") || c.IsSet("maxmemory") {
+			log.Warn("To limit CPU/Memory for function with executor type \"poolmgr\", please specify resources limits when creating environment")
+		}
+		strategy = &fv1.InvokeStrategy{
+			StrategyType: fv1.StrategyTypeExecution,
+			ExecutionStrategy: fv1.ExecutionStrategy{
+				ExecutorType: types.ExecutorTypePoolmgr,
+			},
+		}
+	} else {
+		// set default value
+		targetCPU := DEFAULT_TARGET_CPU_PERCENTAGE
+		minScale := DEFAULT_MIN_SCALE
+		maxScale := minScale
+
+		if existingInvokeStrategy != nil && existingInvokeStrategy.ExecutionStrategy.ExecutorType == types.ExecutorTypeNewdeploy {
+			minScale = existingInvokeStrategy.ExecutionStrategy.MinScale
+			maxScale = existingInvokeStrategy.ExecutionStrategy.MaxScale
+			targetCPU = existingInvokeStrategy.ExecutionStrategy.TargetCPUPercent
+		}
+
+		if c.IsSet("targetcpu") {
+			targetCPU = getTargetCPU(c)
+		}
+
+		if c.IsSet("minscale") {
+			minScale = c.Int("minscale")
+		}
+
+		if c.IsSet("maxscale") {
+			maxScale = c.Int("maxscale")
+			if maxScale <= 0 {
+				return nil, errors.New("Maxscale must be greater than 0")
+			}
+		}
+
+		if minScale > maxScale {
+			return nil, errors.New(fmt.Sprintf("Minscale provided: %v can not be greater than maxscale value %v", minScale, maxScale))
+		}
+
+		// Right now a simple single case strategy implementation
+		// This will potentially get more sophisticated once we have more strategies in place
+		strategy = &fv1.InvokeStrategy{
+			StrategyType: fv1.StrategyTypeExecution,
+			ExecutionStrategy: fv1.ExecutionStrategy{
+				ExecutorType:     fnExecutor,
+				MinScale:         minScale,
+				MaxScale:         maxScale,
+				TargetCPUPercent: targetCPU,
+			},
+		}
+	}
+
+	return strategy, nil
+}
+
+func getTargetCPU(c *cli.Context) int {
+	var targetCPU int
+	if c.IsSet("targetcpu") {
+		targetCPU = c.Int("targetcpu")
+		if targetCPU <= 0 || targetCPU > 100 {
+			log.Fatal("TargetCPU must be a value between 1 - 100")
+		}
+	} else {
+		targetCPU = DEFAULT_TARGET_CPU_PERCENTAGE
+	}
+	return targetCPU
+}
+
+// From this change onwards, we mandate that a function should reference a secret, config map and package in its own ns
+func fnCreate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	fnNamespace := c.String("fnNamespace")
+	envNamespace := c.String("envNamespace")
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		log.Fatal("Need --name argument.")
+	}
+
+	// user wants a spec, create a yaml file with package and function
+	spec := false
+	specFile := ""
+	if c.Bool("spec") {
+		spec = true
+		specFile = fmt.Sprintf("function-%v.yaml", fnName)
+	}
+	specDir := getSpecDir(c)
+
+	// check for unique function names within a namespace
+	fnList, err := client.FunctionList(fnNamespace)
+	util.CheckErr(err, "get function list")
+	// check function existence before creating package
+	for _, fn := range fnList {
+		if fn.Metadata.Name == fnName {
+			log.Fatal("A function with the same name already exists.")
+		}
+	}
+	entrypoint := c.String("entrypoint")
+	pkgName := c.String("pkg")
+
+	secretName := c.String("secret")
+	cfgMapName := c.String("configmap")
+
+	invokeStrategy, err := getInvokeStrategy(c, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	resourceReq := getResourceReq(c, apiv1.ResourceRequirements{})
+
+	var pkgMetadata *metav1.ObjectMeta
+	var envName string
+	if len(pkgName) > 0 {
+		// use existing package
+		pkg, err := client.PackageGet(&metav1.ObjectMeta{
+			Namespace: fnNamespace,
+			Name:      pkgName,
+		})
+		util.CheckErr(err, fmt.Sprintf("read package in '%v' in Namespace: %s. Package needs to be present in the same namespace as function", pkgName, fnNamespace))
+		pkgMetadata = &pkg.Metadata
+		envName = pkg.Spec.Environment.Name
+		if envName != c.String("env") {
+			log.Warn("Function's environment is different than package's environment, package's environment will be used for creating function")
+		}
+		envNamespace = pkg.Spec.Environment.Namespace
+	} else {
+		// need to specify environment for creating new package
+		envName = c.String("env")
+		if len(envName) == 0 {
+			log.Fatal("Need --env argument.")
+		}
+
+		// examine existence of given environment. If specs - then spec validate will do it, don't check here.
+		if !spec {
+			_, err := client.EnvironmentGet(&metav1.ObjectMeta{
+				Namespace: envNamespace,
+				Name:      envName,
+			})
+			if err != nil {
+				if e, ok := err.(ferror.Error); ok && e.Code == ferror.ErrorNotFound {
+					log.Warn(fmt.Sprintf("Environment \"%v\" does not exist. Please create the environment before executing the function. \nFor example: `fission env create --name %v --envns %v --image <image>`\n", envName, envName, envNamespace))
+				} else {
+					util.CheckErr(err, "retrieve environment information")
+				}
+			}
+		}
+
+		srcArchiveFiles := c.StringSlice("src")
+		var deployArchiveFiles []string
+		noZip := false
+		code := c.String("code")
+		if len(code) == 0 {
+			deployArchiveFiles = c.StringSlice("deploy")
+		} else {
+			deployArchiveFiles = append(deployArchiveFiles, c.String("code"))
+			noZip = true
+		}
+		// fatal when both src & deploy archive are empty
+		if len(srcArchiveFiles) == 0 && len(deployArchiveFiles) == 0 {
+			log.Fatal("Need --deploy or --src argument.")
+		}
+
+		buildcmd := c.String("buildcmd")
+
+		// create new package in the same namespace as the function.
+		pkgMetadata = createPackage(client, fnNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, specDir, specFile, noZip)
+	}
+
+	var secrets []fv1.SecretReference
+	var cfgmaps []fv1.ConfigMapReference
+
+	if len(secretName) > 0 {
+		// check the referenced secret is in the same ns as the function, if not give a warning.
+		_, err := client.SecretGet(&metav1.ObjectMeta{
+			Namespace: fnNamespace,
+			Name:      secretName,
+		})
+		if k8serrors.IsNotFound(err) {
+			log.Warn(fmt.Sprintf("Secret %s not found in Namespace: %s. Secret needs to be present in the same namespace as function", secretName, fnNamespace))
+		}
+
+		newSecret := fv1.SecretReference{
+			Name:      secretName,
+			Namespace: fnNamespace,
+		}
+		secrets = []fv1.SecretReference{newSecret}
+	}
+
+	if len(cfgMapName) > 0 {
+		// check the referenced cfgmap is in the same ns as the function, if not give a warning.
+		_, err := client.ConfigMapGet(&metav1.ObjectMeta{
+			Namespace: fnNamespace,
+			Name:      cfgMapName,
+		})
+		if k8serrors.IsNotFound(err) {
+			log.Warn(fmt.Sprintf("ConfigMap %s not found in Namespace: %s. ConfigMap needs to be present in the same namespace as function", cfgMapName, fnNamespace))
+		}
+
+		newCfgMap := fv1.ConfigMapReference{
+			Name:      cfgMapName,
+			Namespace: fnNamespace,
+		}
+		cfgmaps = []fv1.ConfigMapReference{newCfgMap}
+	}
+
+	function := &fv1.Function{
+		Metadata: metav1.ObjectMeta{
+			Name:      fnName,
+			Namespace: fnNamespace,
+		},
+		Spec: fv1.FunctionSpec{
+			Environment: fv1.EnvironmentReference{
+				Name:      envName,
+				Namespace: envNamespace,
+			},
+			Package: fv1.FunctionPackageRef{
+				FunctionName: entrypoint,
+				PackageRef: fv1.PackageRef{
+					Namespace:       pkgMetadata.Namespace,
+					Name:            pkgMetadata.Name,
+					ResourceVersion: pkgMetadata.ResourceVersion,
+				},
+			},
+			Secrets:        secrets,
+			ConfigMaps:     cfgmaps,
+			Resources:      resourceReq,
+			InvokeStrategy: *invokeStrategy,
+		},
+	}
+
+	// if we're writing a spec, don't create the function
+	if spec {
+		err = specSave(*function, specFile)
+		util.CheckErr(err, "create function spec")
+		return nil
+
+	}
+
+	_, err = client.FunctionCreate(function)
+	util.CheckErr(err, "create function")
+
+	fmt.Printf("function '%v' created\n", fnName)
+
+	// Allow the user to specify an HTTP trigger while creating a function.
+	triggerUrl := c.String("url")
+	if len(triggerUrl) == 0 {
+		return nil
+	}
+	if !strings.HasPrefix(triggerUrl, "/") {
+		triggerUrl = fmt.Sprintf("/%s", triggerUrl)
+	}
+
+	method := c.String("method")
+	if len(method) == 0 {
+		method = http.MethodGet
+	}
+	triggerName := uuid.NewV4().String()
+	ht := &fv1.HTTPTrigger{
+		Metadata: metav1.ObjectMeta{
+			Name:      triggerName,
+			Namespace: fnNamespace,
+		},
+		Spec: fv1.HTTPTriggerSpec{
+			RelativeURL: triggerUrl,
+			Method:      getMethod(method),
+			FunctionReference: fv1.FunctionReference{
+				Type: fv1.FunctionReferenceTypeFunctionName,
+				Name: fnName,
+			},
+		},
+	}
+	_, err = client.HTTPTriggerCreate(ht)
+	util.CheckErr(err, "create HTTP trigger")
+	fmt.Printf("route created: %v %v -> %v\n", method, triggerUrl, fnName)
+
+	return err
+}
+
+func fnGet(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		log.Fatal("Need name of function, use --name")
+	}
+	fnNamespace := c.String("fnNamespace")
+	m := &metav1.ObjectMeta{
+		Name:      fnName,
+		Namespace: fnNamespace,
+	}
+	fn, err := client.FunctionGet(m)
+	util.CheckErr(err, "get function")
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Name:      fn.Spec.Package.PackageRef.Name,
+		Namespace: fn.Spec.Package.PackageRef.Namespace,
+	})
+	util.CheckErr(err, "get package")
+
+	os.Stdout.Write(pkg.Spec.Deployment.Literal)
+	return err
+}
+
+func fnGetMeta(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		log.Fatal("Need name of function, use --name")
+	}
+	fnNamespace := c.String("fnNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      fnName,
+		Namespace: fnNamespace,
+	}
+
+	f, err := client.FunctionGet(m)
+	util.CheckErr(err, "get function")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "ENV")
+	fmt.Fprintf(w, "%v\t%v\t%v\n",
+		f.Metadata.Name, f.Metadata.UID, f.Spec.Environment.Name)
+	w.Flush()
+	return err
+}
+
+func fnUpdate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	if len(c.String("package")) > 0 {
+		log.Fatal("--package is deprecated, please use --deploy instead.")
+	}
+
+	if len(c.String("srcpkg")) > 0 {
+		log.Fatal("--srcpkg is deprecated, please use --src instead.")
+	}
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		log.Fatal("Need name of function, use --name")
+	}
+	fnNamespace := c.String("fnNamespace")
+
+	function, err := client.FunctionGet(&metav1.ObjectMeta{
+		Name:      fnName,
+		Namespace: fnNamespace,
+	})
+	util.CheckErr(err, fmt.Sprintf("read function '%v'", fnName))
+
+	envName := c.String("env")
+	envNamespace := c.String("envNamespace")
+	// if the new env specified is the same as the old one, no need to update package
+	// same is true for all update parameters, but, for now, we dont check all of them - because, its ok to
+	// re-write the object with same old values, we just end up getting a new resource version for the object.
+	if len(envName) > 0 && envName == function.Spec.Environment.Name {
+		envName = ""
+	}
+
+	if envNamespace == function.Spec.Environment.Namespace {
+		envNamespace = ""
+	}
+
+	var deployArchiveFiles []string
+	codeFlag := false
+	code := c.String("code")
+	if len(code) == 0 {
+		deployArchiveFiles = c.StringSlice("deploy")
+	} else {
+		deployArchiveFiles = append(deployArchiveFiles, c.String("code"))
+		codeFlag = true
+	}
+
+	srcArchiveFiles := c.StringSlice("src")
+	pkgName := c.String("pkg")
+	entrypoint := c.String("entrypoint")
+	buildcmd := c.String("buildcmd")
+	force := c.Bool("force")
+
+	secretName := c.String("secret")
+	cfgMapName := c.String("configmap")
+
+	if len(srcArchiveFiles) > 0 && len(deployArchiveFiles) > 0 {
+		log.Fatal("Need either of --src or --deploy and not both arguments.")
+	}
+
+	if len(secretName) > 0 {
+		if len(function.Spec.Secrets) > 1 {
+			log.Fatal("Please use 'fission spec apply' to update list of secrets")
+		}
+
+		// check that the referenced secret is in the same ns as the function, if not give a warning.
+		_, err := client.SecretGet(&metav1.ObjectMeta{
+			Namespace: fnNamespace,
+			Name:      secretName,
+		})
+		if k8serrors.IsNotFound(err) {
+			log.Warn(fmt.Sprintf("secret %s not found in Namespace: %s. Secret needs to be present in the same namespace as function", secretName, fnNamespace))
+		}
+
+		newSecret := fv1.SecretReference{
+			Name:      secretName,
+			Namespace: fnNamespace,
+		}
+		function.Spec.Secrets = []fv1.SecretReference{newSecret}
+	}
+
+	if len(cfgMapName) > 0 {
+		if len(function.Spec.ConfigMaps) > 1 {
+			log.Fatal("Please use 'fission spec apply' to update list of configmaps")
+		}
+
+		// check that the referenced cfgmap is in the same ns as the function, if not give a warning.
+		_, err := client.ConfigMapGet(&metav1.ObjectMeta{
+			Namespace: fnNamespace,
+			Name:      cfgMapName,
+		})
+		if k8serrors.IsNotFound(err) {
+			log.Warn(fmt.Sprintf("ConfigMap %s not found in Namespace: %s. ConfigMap needs to be present in the same namespace as the function", cfgMapName, fnNamespace))
+		}
+
+		newCfgMap := fv1.ConfigMapReference{
+			Name:      cfgMapName,
+			Namespace: fnNamespace,
+		}
+		function.Spec.ConfigMaps = []fv1.ConfigMapReference{newCfgMap}
+	}
+
+	if len(envName) > 0 {
+		function.Spec.Environment.Name = envName
+	}
+
+	if len(envNamespace) > 0 {
+		function.Spec.Environment.Namespace = envNamespace
+	}
+
+	if len(entrypoint) > 0 {
+		function.Spec.Package.FunctionName = entrypoint
+	}
+	if len(pkgName) == 0 {
+		pkgName = function.Spec.Package.PackageRef.Name
+	}
+
+	strategy, err := getInvokeStrategy(c, &function.Spec.InvokeStrategy)
+	if err != nil {
+		log.Fatal(err)
+	}
+	function.Spec.InvokeStrategy = *strategy
+	function.Spec.Resources = getResourceReq(c, function.Spec.Resources)
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: fnNamespace,
+		Name:      pkgName,
+	})
+	util.CheckErr(err, fmt.Sprintf("read package '%v.%v'. Pkg should be present in the same ns as the function", pkgName, fnNamespace))
+
+	pkgMetadata := &pkg.Metadata
+
+	if len(deployArchiveFiles) != 0 || len(srcArchiveFiles) != 0 || len(buildcmd) != 0 || len(envName) != 0 || len(envNamespace) != 0 {
+		fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name, pkg.Metadata.Namespace)
+		util.CheckErr(err, "get function list")
+
+		if !force && len(fnList) > 1 {
+			log.Fatal("Package is used by multiple functions, use --force to force update")
+		}
+
+		pkgMetadata, err = updatePackage(client, pkg, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, false, codeFlag)
+		util.CheckErr(err, fmt.Sprintf("update package '%v'", pkgName))
+
+		fmt.Printf("package '%v' updated\n", pkgMetadata.GetName())
+
+		// update resource version of package reference of functions that shared the same package
+		for _, fn := range fnList {
+			// ignore the update for current function here, it will be updated later.
+			if fn.Metadata.Name != fnName {
+				fn.Spec.Package.PackageRef.ResourceVersion = pkgMetadata.ResourceVersion
+				_, err := client.FunctionUpdate(&fn)
+				util.CheckErr(err, "update function")
+			}
+		}
+	}
+
+	// TODO : One corner case where user just updates the pkg reference with fnUpdate, but internally this new pkg reference
+	// references a diff env than the spec
+
+	// update function spec with new package metadata
+	function.Spec.Package.PackageRef = fv1.PackageRef{
+		Namespace:       pkgMetadata.Namespace,
+		Name:            pkgMetadata.Name,
+		ResourceVersion: pkgMetadata.ResourceVersion,
+	}
+
+	if function.Spec.Environment.Name != pkg.Spec.Environment.Name {
+		log.Warn("Function's environment is different than package's environment, package's environment will be used for updating function")
+		function.Spec.Environment.Name = pkg.Spec.Environment.Name
+		function.Spec.Environment.Namespace = pkg.Spec.Environment.Namespace
+	}
+
+	_, err = client.FunctionUpdate(function)
+	util.CheckErr(err, "update function")
+
+	fmt.Printf("function '%v' updated\n", fnName)
+	return err
+}
+
+func fnDelete(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		log.Fatal("Need name of function, use --name")
+	}
+	fnNamespace := c.String("fnNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      fnName,
+		Namespace: fnNamespace,
+	}
+
+	err := client.FunctionDelete(m)
+	util.CheckErr(err, fmt.Sprintf("delete function '%v'", fnName))
+
+	fmt.Printf("function '%v' deleted\n", fnName)
+	return err
+}
+
+func fnList(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+	ns := c.String("fnNamespace")
+
+	fns, err := client.FunctionList(ns)
+	util.CheckErr(err, "list functions")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "ENV", "EXECUTORTYPE", "MINSCALE", "MAXSCALE", "MINCPU", "MAXCPU", "MINMEMORY", "MAXMEMORY", "TARGETCPU")
+	for _, f := range fns {
+		mincpu := f.Spec.Resources.Requests.Cpu
+		mincpu().Value()
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+			f.Metadata.Name, f.Metadata.UID, f.Spec.Environment.Name,
+			f.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType,
+			f.Spec.InvokeStrategy.ExecutionStrategy.MinScale,
+			f.Spec.InvokeStrategy.ExecutionStrategy.MaxScale,
+			f.Spec.Resources.Requests.Cpu().String(),
+			f.Spec.Resources.Limits.Cpu().String(),
+			f.Spec.Resources.Requests.Memory().String(),
+			f.Spec.Resources.Limits.Memory().String(),
+			f.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent)
+	}
+	w.Flush()
+
+	return err
+}
+
+func fnLogs(c *cli.Context) error {
+
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		log.Fatal("Need name of function, use --name")
+	}
+	fnNamespace := c.String("fnNamespace")
+
+	dbType := c.String("dbtype")
+	if len(dbType) == 0 {
+		dbType = logdb.INFLUXDB
+	}
+
+	fnPod := c.String("pod")
+	m := &metav1.ObjectMeta{
+		Name:      fnName,
+		Namespace: fnNamespace,
+	}
+
+	recordLimit := c.Int("recordcount")
+	if recordLimit <= 0 {
+		recordLimit = 1000
+	}
+
+	f, err := client.FunctionGet(m)
+	util.CheckErr(err, "get function")
+
+	// request the controller to establish a proxy server to the database.
+	logDB, err := logdb.GetLogDB(dbType, util.GetServerUrl())
+	if err != nil {
+		log.Fatal("failed to connect log database")
+	}
+
+	requestChan := make(chan struct{})
+	responseChan := make(chan struct{})
+	ctx := context.Background()
+
+	go func(ctx context.Context, requestChan, responseChan chan struct{}) {
+		t := time.Unix(0, 0*int64(time.Millisecond))
+		for {
+			select {
+			case <-requestChan:
+				logFilter := logdb.LogFilter{
+					Pod:         fnPod,
+					Function:    f.Metadata.Name,
+					FuncUid:     string(f.Metadata.UID),
+					Since:       t,
+					RecordLimit: recordLimit,
+				}
+				logEntries, err := logDB.GetLogs(logFilter)
+				if err != nil {
+					log.Fatal(fmt.Sprintf("Error querying logs: %v", err))
+				}
+				for _, logEntry := range logEntries {
+					if c.Bool("d") {
+						fmt.Printf("Timestamp: %s\nNamespace: %s\nFunction Name: %s\nFunction ID: %s\nPod: %s\nContainer: %s\nStream: %s\nLog: %s\n---\n",
+							logEntry.Timestamp, logEntry.Namespace, logEntry.FuncName, logEntry.FuncUid, logEntry.Pod, logEntry.Container, logEntry.Stream, logEntry.Message)
+					} else {
+						fmt.Printf("[%s] %s\n", logEntry.Timestamp, logEntry.Message)
+					}
+					t = logEntry.Timestamp
+				}
+				responseChan <- struct{}{}
+			case <-ctx.Done():
+				return
+			}
+		}
+	}(ctx, requestChan, responseChan)
+
+	for {
+		requestChan <- struct{}{}
+		<-responseChan
+		if !c.Bool("f") {
+			ctx.Done()
+			return nil
+		}
+		time.Sleep(1 * time.Second)
+	}
+}
+
+func fnTest(c *cli.Context) error {
+	fnName := c.String("name")
+	if len(fnName) == 0 {
+		log.Fatal("Need function name to be specified with --name")
+	}
+	ns := c.String("fnNamespace")
+
+	routerURL := os.Getenv("FISSION_ROUTER")
+	if len(routerURL) == 0 {
+		// Portforward to the fission router
+		localRouterPort := util.SetupPortForward(util.GetFissionNamespace(),
+			"application=fission-router")
+		routerURL = "127.0.0.1:" + localRouterPort
+	} else {
+		routerURL = strings.TrimPrefix(routerURL, "http://")
+	}
+
+	fnUri := fnName
+	if ns != metav1.NamespaceDefault {
+		fnUri = fmt.Sprintf("%v/%v", ns, fnName)
+	}
+
+	functionUrl, err := url.Parse(fmt.Sprintf("http://%s/fission-function/%s", routerURL, fnUri))
+	if err != nil {
+		log.Fatal(err)
+	}
+	queryParams := c.StringSlice("query")
+	if len(queryParams) > 0 {
+		query := url.Values{}
+		for _, q := range queryParams {
+			queryParts := strings.SplitN(q, "=", 2)
+			var key, value string
+			if len(queryParts) == 0 {
+				continue
+			}
+			if len(queryParts) > 0 {
+				key = queryParts[0]
+			}
+			if len(queryParts) > 1 {
+				value = queryParts[1]
+			}
+			query.Set(key, value)
+		}
+		functionUrl.RawQuery = query.Encode()
+	}
+
+	ctx := context.Background()
+	if deadline := c.Duration("timeout"); deadline > 0 {
+		var closeCtx func()
+		ctx, closeCtx = context.WithTimeout(ctx, deadline)
+		defer closeCtx()
+	}
+
+	headers := c.StringSlice("header")
+
+	resp := doHTTPRequest(ctx, c.String("method"), functionUrl.String(), c.String("body"), headers)
+	if resp.StatusCode < 400 {
+		body, err := ioutil.ReadAll(resp.Body)
+		util.CheckErr(err, "Function test")
+		fmt.Print(string(body))
+		defer resp.Body.Close()
+		return nil
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	util.CheckErr(err, "read log response from pod")
+	fmt.Printf("Error calling function %s: %d; Please try again or fix the error: %s", fnName, resp.StatusCode, string(body))
+	defer resp.Body.Close()
+	err = printPodLogs(c)
+	if err != nil {
+		fnLogs(c)
+	}
+
+	return nil
+}
+
+func doHTTPRequest(ctx context.Context, method, url, body string, headers []string) *http.Response {
+	if method == "" {
+		method = http.MethodGet
+	}
+
+	if method != http.MethodGet &&
+		method != http.MethodDelete &&
+		method != http.MethodPost &&
+		method != http.MethodPut &&
+		method != http.MethodOptions {
+		log.Fatal(fmt.Sprintf("Invalid HTTP method '%s'.", method))
+	}
+
+	req, err := http.NewRequest(method, url, strings.NewReader(body))
+	util.CheckErr(err, "create HTTP request")
+
+	for _, header := range headers {
+		headerKeyValue := strings.SplitN(header, ":", 2)
+		if len(headerKeyValue) != 2 {
+			log.Fatal("Failed to create request without appropriate headers")
+		}
+		req.Header.Set(headerKeyValue[0], headerKeyValue[1])
+	}
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	util.CheckErr(err, "execute HTTP request")
+
+	return resp
+}

pkg/fission-cli/function_test.go

@@ -0,0 +1,289 @@
+package fission_cli
+
+import (
+	"flag"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+)
+
+func TestGetInvokeStrategy(t *testing.T) {
+	cases := []struct {
+		testArgs               map[string]string
+		existingInvokeStrategy *fv1.InvokeStrategy
+		expectedResult         *fv1.InvokeStrategy
+		expectError            bool
+	}{
+		{
+			// case: use default executor poolmgr
+			testArgs:               map[string]string{},
+			existingInvokeStrategy: nil,
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType: fv1.ExecutorTypePoolmgr,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: executor type set to poolmgr
+			testArgs:               map[string]string{"executortype": fv1.ExecutorTypePoolmgr},
+			existingInvokeStrategy: nil,
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType: fv1.ExecutorTypePoolmgr,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: executor type set to newdeploy
+			testArgs:               map[string]string{"executortype": fv1.ExecutorTypeNewdeploy},
+			existingInvokeStrategy: nil,
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         DEFAULT_MIN_SCALE,
+					MaxScale:         DEFAULT_MIN_SCALE,
+					TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: executor type change from poolmgr to newdeploy
+			testArgs: map[string]string{"executortype": fv1.ExecutorTypeNewdeploy},
+			existingInvokeStrategy: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType: fv1.ExecutorTypePoolmgr,
+				},
+			},
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         DEFAULT_MIN_SCALE,
+					MaxScale:         DEFAULT_MIN_SCALE,
+					TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: executor type change from newdeploy to poolmgr
+			testArgs: map[string]string{"executortype": fv1.ExecutorTypePoolmgr},
+			existingInvokeStrategy: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         DEFAULT_MIN_SCALE,
+					MaxScale:         DEFAULT_MIN_SCALE,
+					TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE,
+				},
+			},
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType: fv1.ExecutorTypePoolmgr,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: minscale < maxscale
+			testArgs: map[string]string{
+				"executortype": fv1.ExecutorTypeNewdeploy,
+				"minscale":     "2",
+				"maxscale":     "3",
+			},
+			existingInvokeStrategy: nil,
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         2,
+					MaxScale:         3,
+					TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: minscale > maxscale
+			testArgs: map[string]string{
+				"executortype": fv1.ExecutorTypeNewdeploy,
+				"minscale":     "5",
+				"maxscale":     "3",
+			},
+			existingInvokeStrategy: nil,
+			expectedResult:         nil,
+			expectError:            true,
+		},
+		{
+			// case: maxscale not specified
+			testArgs: map[string]string{
+				"executortype": fv1.ExecutorTypeNewdeploy,
+				"minscale":     "5",
+			},
+			existingInvokeStrategy: nil,
+			expectedResult:         nil,
+			expectError:            true,
+		},
+		{
+			// case: minscale not specified
+			testArgs: map[string]string{
+				"executortype": fv1.ExecutorTypeNewdeploy,
+				"maxscale":     "3",
+			},
+			existingInvokeStrategy: nil,
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         DEFAULT_MIN_SCALE,
+					MaxScale:         3,
+					TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: maxscale set to 0
+			testArgs: map[string]string{
+				"executortype": fv1.ExecutorTypeNewdeploy,
+				"maxscale":     "0",
+			},
+			existingInvokeStrategy: nil,
+			expectedResult:         nil,
+			expectError:            true,
+		},
+		{
+			// case: maxscale set to 9 when existing is 5
+			testArgs: map[string]string{
+				"executortype": fv1.ExecutorTypeNewdeploy,
+				"maxscale":     "9",
+			},
+			existingInvokeStrategy: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         2,
+					MaxScale:         5,
+					TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE,
+				},
+			},
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         2,
+					MaxScale:         9,
+					TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: change nothing for existing strategy
+			testArgs: map[string]string{
+				"executortype": fv1.ExecutorTypeNewdeploy,
+			},
+			existingInvokeStrategy: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         2,
+					MaxScale:         5,
+					TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE,
+				},
+			},
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         2,
+					MaxScale:         5,
+					TargetCPUPercent: DEFAULT_TARGET_CPU_PERCENTAGE,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: set target cpu percentage
+			testArgs: map[string]string{
+				"executortype": fv1.ExecutorTypeNewdeploy,
+				"targetcpu":    "50",
+			},
+			existingInvokeStrategy: nil,
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         DEFAULT_MIN_SCALE,
+					MaxScale:         DEFAULT_MIN_SCALE,
+					TargetCPUPercent: 50,
+				},
+			},
+			expectError: false,
+		},
+		{
+			// case: change target cpu percentage
+			testArgs: map[string]string{
+				"executortype": fv1.ExecutorTypeNewdeploy,
+				"targetcpu":    "20",
+			},
+			existingInvokeStrategy: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         2,
+					MaxScale:         5,
+					TargetCPUPercent: 88,
+				},
+			},
+			expectedResult: &fv1.InvokeStrategy{
+				StrategyType: fv1.StrategyTypeExecution,
+				ExecutionStrategy: fv1.ExecutionStrategy{
+					ExecutorType:     fv1.ExecutorTypeNewdeploy,
+					MinScale:         2,
+					MaxScale:         5,
+					TargetCPUPercent: 20,
+				},
+			},
+			expectError: false,
+		},
+	}
+
+	for i, c := range cases {
+		fmt.Printf("=== Test Case %v ===\n", i)
+
+		app := NewCliApp()
+		set := flag.NewFlagSet("test-cmd", 0)
+		ctx := cli.NewContext(app, set, nil)
+
+		for k, v := range c.testArgs {
+			set.String(k, v, "")
+			ctx.Set(k, v)
+		}
+
+		strategy, err := getInvokeStrategy(ctx, c.existingInvokeStrategy)
+		if c.expectError {
+			assert.NotNil(t, err)
+			if err != nil {
+				fmt.Println(err)
+			}
+		} else {
+			assert.Nil(t, err)
+			assert.NoError(t, strategy.Validate(), fmt.Sprintf("Failed at test case %v", i))
+			assert.Equal(t, *c.expectedResult, *strategy)
+		}
+	}
+}

pkg/fission-cli/httptrigger.go

@@ -0,0 +1,306 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/satori/go.uuid"
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/util"
+)
+
+// returns one of http.Method*
+func getMethod(method string) string {
+	switch strings.ToUpper(method) {
+	case "GET":
+		return http.MethodGet
+	case "HEAD":
+		return http.MethodHead
+	case "POST":
+		return http.MethodPost
+	case "PUT":
+		return http.MethodPut
+	case "PATCH":
+		return http.MethodPatch
+	case "DELETE":
+		return http.MethodDelete
+	case "CONNECT":
+		return http.MethodConnect
+	case "OPTIONS":
+		return http.MethodOptions
+	case "TRACE":
+		return http.MethodTrace
+	}
+	log.Fatal(fmt.Sprintf("Invalid HTTP Method %v", method))
+	return ""
+}
+
+func setHtFunctionRef(functionList []string, functionWeightsList []int) (*fv1.FunctionReference, error) {
+	if len(functionList) == 1 {
+		return &fv1.FunctionReference{
+			Type: fv1.FunctionReferenceTypeFunctionName,
+			Name: functionList[0],
+		}, nil
+	} else if len(functionList) == 2 {
+		if len(functionWeightsList) != 2 {
+			return nil, fmt.Errorf("weights of the function need to be specified when 2 functions are supplied")
+		}
+
+		totalWeight := functionWeightsList[0] + functionWeightsList[1]
+		if totalWeight != 100 {
+			log.Fatal("The function weights should add up to 100")
+		}
+
+		functionWeights := make(map[string]int, 0)
+		for index := range functionList {
+			functionWeights[functionList[index]] = functionWeightsList[index]
+		}
+
+		return &fv1.FunctionReference{
+			Type:            fv1.FunctionReferenceTypeFunctionWeights,
+			FunctionWeights: functionWeights,
+		}, nil
+	}
+
+	return nil, fmt.Errorf("the number of functions in a trigger can be 1 or 2(for canary feature along with their weights)")
+}
+
+func htCreate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	functionList := c.StringSlice("function")
+	functionWeightsList := c.IntSlice("weight")
+
+	if len(functionList) == 0 {
+		log.Fatal("Need a function name to create a trigger, use --function")
+	}
+
+	functionRef, err := setHtFunctionRef(functionList, functionWeightsList)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+
+	triggerName := c.String("name")
+	fnNamespace := c.String("fnNamespace")
+	spec := c.Bool("spec")
+
+	m := &metav1.ObjectMeta{
+		Name:      triggerName,
+		Namespace: fnNamespace,
+	}
+
+	htTrigger, err := client.HTTPTriggerGet(m)
+	if htTrigger != nil {
+		util.CheckErr(fmt.Errorf("duplicate trigger exists"), "choose a different name or leave it empty for fission to auto-generate it")
+	}
+
+	triggerUrl := c.String("url")
+	if len(triggerUrl) == 0 {
+		log.Fatal("Need a trigger URL, use --url")
+	}
+	if !strings.HasPrefix(triggerUrl, "/") {
+		triggerUrl = fmt.Sprintf("/%s", triggerUrl)
+	}
+
+	method := c.String("method")
+	if len(method) == 0 {
+		method = "GET"
+	}
+
+	// For Specs, the spec validate checks for function reference
+	if !spec {
+		err = util.CheckFunctionExistence(client, functionList, fnNamespace)
+		if err != nil {
+			log.Warn(err.Error())
+		}
+	}
+
+	createIngress := false
+	if c.IsSet("createingress") {
+		createIngress = c.Bool("createingress")
+	}
+
+	host := c.String("host")
+
+	// just name triggers by uuid.
+	if triggerName == "" {
+		triggerName = uuid.NewV4().String()
+	}
+
+	ht := &fv1.HTTPTrigger{
+		Metadata: metav1.ObjectMeta{
+			Name:      triggerName,
+			Namespace: fnNamespace,
+		},
+		Spec: fv1.HTTPTriggerSpec{
+			Host:              host,
+			RelativeURL:       triggerUrl,
+			Method:            getMethod(method),
+			FunctionReference: *functionRef,
+			CreateIngress:     createIngress,
+		},
+	}
+
+	// if we're writing a spec, don't call the API
+	if spec {
+		specFile := fmt.Sprintf("route-%v.yaml", triggerName)
+		err := specSave(*ht, specFile)
+		util.CheckErr(err, "create HTTP trigger spec")
+		return nil
+	}
+
+	_, err = client.HTTPTriggerCreate(ht)
+	util.CheckErr(err, "create HTTP trigger")
+
+	fmt.Printf("trigger '%v' created\n", triggerName)
+	return err
+}
+
+func htGet(c *cli.Context) error {
+	cliClient := util.GetApiClient(c.GlobalString("server"))
+
+	name := c.String("name")
+	ns := c.String("fnNamespace")
+
+	m := &metav1.ObjectMeta{
+		Name:      name,
+		Namespace: ns,
+	}
+
+	htTrigger, err := cliClient.HTTPTriggerGet(m)
+	util.CheckErr(err, "get http trigger")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 1, 1, ' ', 0)
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "METHOD", "RELATIVE-URL", "FUNCTION-REFERENCE-TYPE", "FUNCTION(s)")
+
+	function := ""
+	if htTrigger.Spec.FunctionReference.Type == fv1.FunctionReferenceTypeFunctionName {
+		function = htTrigger.Spec.FunctionReference.Name
+	} else {
+		for k, v := range htTrigger.Spec.FunctionReference.FunctionWeights {
+			function += fmt.Sprintf("%s:%v ", k, v)
+		}
+	}
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n",
+		htTrigger.Metadata.Name, htTrigger.Metadata.UID, htTrigger.Spec.Method, htTrigger.Spec.RelativeURL,
+		htTrigger.Spec.FunctionReference.Type, function)
+
+	w.Flush()
+
+	return err
+}
+
+func htUpdate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+	htName := c.String("name")
+	if len(htName) == 0 {
+		log.Fatal("Need name of trigger, use --name")
+	}
+	triggerNamespace := c.String("triggerNamespace")
+
+	ht, err := client.HTTPTriggerGet(&metav1.ObjectMeta{
+		Name:      htName,
+		Namespace: triggerNamespace,
+	})
+	util.CheckErr(err, "get HTTP trigger")
+
+	if c.IsSet("function") {
+		// get the functions and their weights if specified
+		functionList := c.StringSlice("function")
+		err := util.CheckFunctionExistence(client, functionList, triggerNamespace)
+		if err != nil {
+			if err != nil {
+				log.Warn(err.Error())
+			}
+		}
+
+		var functionWeightsList []int
+		if c.IsSet("weight") {
+			functionWeightsList = c.IntSlice("weight")
+		}
+
+		// set function reference
+		functionRef, err := setHtFunctionRef(functionList, functionWeightsList)
+		if err != nil {
+			log.Fatal(err.Error())
+		}
+
+		ht.Spec.FunctionReference = *functionRef
+	}
+
+	if c.IsSet("createingress") {
+		ht.Spec.CreateIngress = c.Bool("createingress")
+	}
+
+	if c.IsSet("host") {
+		ht.Spec.Host = c.String("host")
+	}
+
+	_, err = client.HTTPTriggerUpdate(ht)
+	util.CheckErr(err, "update HTTP trigger")
+
+	fmt.Printf("trigger '%v' updated\n", htName)
+	return nil
+}
+
+func htDelete(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+	htName := c.String("name")
+	if len(htName) == 0 {
+		log.Fatal("Need name of trigger to delete, use --name")
+	}
+	triggerNamespace := c.String("triggerNamespace")
+
+	err := client.HTTPTriggerDelete(&metav1.ObjectMeta{
+		Name:      htName,
+		Namespace: triggerNamespace,
+	})
+	util.CheckErr(err, "delete trigger")
+
+	fmt.Printf("trigger '%v' deleted\n", htName)
+	return nil
+}
+
+func htList(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+	triggerNamespace := c.String("triggerNamespace")
+
+	hts, err := client.HTTPTriggerList(triggerNamespace)
+	util.CheckErr(err, "list HTTP triggers")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "METHOD", "HOST", "URL", "INGRESS", "FUNCTION_NAME")
+	for _, ht := range hts {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n",
+			ht.Metadata.Name, ht.Spec.Method, ht.Spec.Host, ht.Spec.RelativeURL, ht.Spec.CreateIngress, ht.Spec.FunctionReference.Name)
+	}
+	w.Flush()
+
+	return nil
+}

pkg/fission-cli/log/log.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package log
+
+import (
+	"fmt"
+	"os"
+)
+
+var (
+	// global Verbosity of our CLI
+	Verbosity int
+)
+
+func Fatal(msg interface{}) {
+	os.Stderr.WriteString(fmt.Sprintf("Fatal error: %v\n", msg))
+	os.Exit(1)
+}
+
+func Warn(msg interface{}) {
+	os.Stderr.WriteString(fmt.Sprintf("Warning: %v\n", msg))
+}
+
+func Info(msg interface{}) {
+	os.Stderr.WriteString(fmt.Sprintf("%v\n", msg))
+}
+
+func Verbose(verbosityLevel int, format string, args ...interface{}) {
+	if Verbosity >= verbosityLevel {
+		fmt.Printf(format+"\n", args...)
+	}
+}

pkg/fission-cli/logdb/influxdb.go

@@ -0,0 +1,202 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logdb
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	influxdbClient "github.com/influxdata/influxdb/client/v2"
+
+	ferror "github.com/fission/fission/pkg/error"
+	"github.com/fission/fission/pkg/fission-cli/log"
+)
+
+const (
+	INFLUXDB_DATABASE = "fissionFunctionLog"
+	INFLUXDB_URL      = "http://influxdb:8086/query"
+)
+
+func NewInfluxDB(serverURL string) (InfluxDB, error) {
+	return InfluxDB{endpoint: serverURL}, nil
+}
+
+type InfluxDB struct {
+	endpoint string
+}
+
+func makeIndexMap(cols []string) map[string]int {
+	indexMap := make(map[string]int, len(cols))
+	for i := range cols {
+		indexMap[cols[i]] = i
+	}
+
+	return indexMap
+}
+
+func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
+	timestamp := filter.Since.UnixNano()
+	var queryCmd string
+
+	// please check "Example 4: Bind a parameter in the WHERE clause to specific tag value"
+	// at https://docs.influxdata.com/influxdb/v1.2/tools/api/
+	parameters := make(map[string]interface{})
+	parameters["funcuid"] = filter.FuncUid
+	parameters["time"] = timestamp
+	//the parameters above are only for the where clause and do not work with LIMIT
+
+	if filter.Pod != "" {
+		// wait for bug fix for fluent-bit influxdb plugin
+		queryCmd = "select * from /^log*/ where (\"funcuid\" = $funcuid OR \"kubernetes_labels_functionUid\" = $funcuid) AND \"pod\" = $pod AND \"time\" > $time LIMIT " + strconv.Itoa(filter.RecordLimit)
+		parameters["pod"] = filter.Pod
+	} else {
+		// wait for bug fix for fluent-bit influxdb plugin
+		queryCmd = "select * from /^log*/ where (\"funcuid\" = $funcuid  OR \"kubernetes_labels_functionUid\" = $funcuid) AND \"time\" > $time LIMIT " + strconv.Itoa(filter.RecordLimit)
+	}
+
+	query := influxdbClient.NewQueryWithParameters(queryCmd, INFLUXDB_DATABASE, "", parameters)
+	logEntries := []LogEntry{}
+	response, err := influx.query(query)
+	if err != nil {
+		return logEntries, err
+	}
+	for _, r := range response.Results {
+		for _, series := range r.Series {
+
+			//create map of columns to row indeces
+			indexMap := makeIndexMap(series.Columns)
+
+			// TODO: Remove fallback indexes. Some of index's name changed in fluent-bit, here we add extra fallbackIndexes to address compatibility problem.
+			container := indexMap["kubernetes_docker_id"]
+			container_1 := indexMap["docker_container_id"] // for backward compatibility
+			functionName := indexMap["kubernetes_labels_functionName"]
+			funcuid := indexMap["kubernetes_labels_functionUid"]
+			funcuid_1 := indexMap["funcuid"]                         // for backward compatibility
+			funcuid_2 := indexMap["kubernetes_labels_functionUid_1"] // for backward compatibility
+			logMessage := indexMap["log"]
+			nameSpace := indexMap["kubernetes_namespace_name"]
+			podName := indexMap["kubernetes_pod_name"]
+			stream := indexMap["stream"]
+			seq := indexMap["_seq"]
+
+			for _, row := range series.Values {
+				t, err := time.Parse(time.RFC3339, row[0].(string))
+				if err != nil {
+					log.Fatal(err)
+				}
+				seqNum, err := strconv.Atoi(row[seq].(string))
+				if err != nil {
+					return logEntries, err
+				}
+				entry := LogEntry{
+					//The attributes of the LogEntry are selected as relative to their position in InfluxDB's line protocol response
+					Timestamp: t,
+					Container: getEntryValue(row, container, container_1),
+					FuncName:  getEntryValue(row, functionName, -1),
+					FuncUid:   getEntryValue(row, funcuid, funcuid_1, funcuid_2),
+					Message:   strings.TrimSuffix(getEntryValue(row, logMessage, -1), "\n"), //log field
+					Namespace: getEntryValue(row, nameSpace, -1),
+					Pod:       getEntryValue(row, podName, -1),
+					Stream:    getEntryValue(row, stream, -1),
+					Sequence:  seqNum, //sequence tag
+				}
+				logEntries = append(logEntries, entry)
+			}
+		}
+	}
+	sort.Slice(logEntries, func(i, j int) bool {
+
+		if logEntries[i].Timestamp.Before(logEntries[j].Timestamp) {
+			return true
+		}
+		if logEntries[j].Timestamp.Before(logEntries[i].Timestamp) {
+			return false
+		}
+		return logEntries[i].Sequence < logEntries[j].Sequence
+	})
+	return logEntries, nil
+}
+
+func (influx InfluxDB) query(query influxdbClient.Query) (*influxdbClient.Response, error) {
+	queryURL, err := url.Parse(influx.endpoint)
+	if err != nil {
+		return nil, err
+	}
+	// connect to controller first, then controller will redirect our query command
+	// to influxdb and proxy back the db response.
+	queryURL.Path = fmt.Sprintf("/proxy/%s", INFLUXDB)
+	req, err := http.NewRequest("POST", queryURL.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	parametersBytes, err := json.Marshal(query.Parameters)
+	if err != nil {
+		return nil, err
+	}
+
+	// set up http URL query string
+	params := req.URL.Query()
+	params.Set("q", query.Command)
+	params.Set("db", query.Database)
+	params.Set("params", string(parametersBytes))
+	req.URL.RawQuery = params.Encode()
+
+	httpClient := http.Client{Timeout: 5 * time.Second}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, ferror.MakeErrorFromHTTP(resp)
+	}
+
+	// decode influxdb response
+	response := influxdbClient.Response{}
+	decoder := json.NewDecoder(resp.Body)
+	decoder.UseNumber()
+	if decoder.Decode(&response) != nil {
+		return nil, fmt.Errorf("Failed to decode influxdb response: %v", err)
+	}
+	return &response, nil
+}
+
+// getEntryValue returns a field value in string type of log entry by providing index of log entry.
+// Since we switch from fluentd to fluent-bit, there are some field names' changed which will break
+// CLI due to empty value. For backward compatibility, getEntryValue also supports to get value from
+// fallbackIndex if exists, otherwise an empty string returned instead.
+func getEntryValue(list []interface{}, index int, fallbackIndex ...int) string {
+	if index < len(list) && list[index] != nil {
+		return list[index].(string)
+	}
+
+	for _, i := range fallbackIndex {
+		if i >= 0 && i < len(list) && list[i] != nil {
+			return list[i].(string)
+		}
+	}
+
+	return ""
+}

pkg/fission-cli/logdb/logdb.go

@@ -0,0 +1,58 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logdb
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	INFLUXDB = "influxdb"
+)
+
+type LogDatabase interface {
+	GetLogs(LogFilter) ([]LogEntry, error)
+}
+
+type LogFilter struct {
+	Pod         string
+	Function    string
+	FuncUid     string
+	Since       time.Time
+	RecordLimit int
+}
+
+type LogEntry struct {
+	Timestamp time.Time
+	Message   string
+	Stream    string
+	Sequence  int
+	Container string
+	Namespace string
+	FuncName  string
+	FuncUid   string
+	Pod       string
+}
+
+func GetLogDB(dbType string, serverURL string) (LogDatabase, error) {
+	switch dbType {
+	case INFLUXDB:
+		return NewInfluxDB(serverURL)
+	}
+	return nil, fmt.Errorf("log database type is incorrect, now only support %s", INFLUXDB)
+}

pkg/fission-cli/main.go

@@ -0,0 +1,467 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/plugin"
+	"github.com/fission/fission/pkg/fission-cli/support"
+	"github.com/fission/fission/pkg/fission-cli/util"
+	"github.com/fission/fission/pkg/info"
+	"github.com/fission/fission/pkg/types"
+)
+
+func cliHook(c *cli.Context) error {
+	log.Verbosity = c.Int("verbosity")
+	log.Verbose(2, "Verbosity = 2")
+
+	err := flagValueParser(c.Args())
+	if err != nil {
+		// The cli package wont't print out error, as a workaround we need to
+		// fatal here instead of return it.
+		log.Fatal(err)
+	}
+
+	return nil
+}
+
+func NewCliApp() *cli.App {
+	app := cli.NewApp()
+	app.Name = "fission"
+	app.Usage = "Serverless functions for Kubernetes"
+	app.Version = info.Version
+	cli.VersionPrinter = versionPrinter
+	app.CustomAppHelpTemplate = helpTemplate
+	app.ExtraInfo = func() map[string]string {
+		info := map[string]string{}
+		for _, pmd := range plugin.FindAll() {
+			names := strings.Join(append([]string{pmd.Name}, pmd.Aliases...), ", ")
+			info[names] = pmd.Usage
+		}
+		return info
+	}
+
+	app.Flags = []cli.Flag{
+		cli.StringFlag{Name: "server", Value: "", Usage: "Fission server URL"},
+		cli.IntFlag{Name: "verbosity", Value: 1, Usage: "CLI verbosity (0 is quiet, 1 is the default, 2 is verbose.)"},
+		cli.BoolFlag{Name: "plugin", Hidden: true},
+	}
+
+	// all resource create commands accept --spec
+	specSaveFlag := cli.BoolFlag{Name: "spec", Usage: "Save to the spec directory instead of creating on cluster"}
+
+	// namespace reference for all objects
+	fnNamespaceFlag := cli.StringFlag{Name: "fnNamespace, fns", Value: metav1.NamespaceDefault, Usage: "Namespace for function object"}
+	envNamespaceFlag := cli.StringFlag{Name: "envNamespace, envns", Value: metav1.NamespaceDefault, Usage: "Namespace for environment object"}
+	pkgNamespaceFlag := cli.StringFlag{Name: "pkgNamespace, pkgns", Value: metav1.NamespaceDefault, Usage: "Namespace for package object"}
+	triggerNamespaceFlag := cli.StringFlag{Name: "triggerNamespace, triggerns", Value: metav1.NamespaceDefault, Usage: "Namespace for trigger object"}
+	recorderNamespaceFlag := cli.StringFlag{Name: "recorderNamespace, recorderns", Value: metav1.NamespaceDefault, Usage: "Namespace for recorder object"}
+	canaryNamespaceFlag := cli.StringFlag{Name: "canaryNamespace, canaryns", Value: metav1.NamespaceDefault, Usage: "Namespace for canary config object"}
+
+	// trigger method and url flags (used in function and route CLIs)
+	htMethodFlag := cli.StringFlag{Name: "method", Value: "GET", Usage: "HTTP Method: GET|POST|PUT|DELETE|HEAD"}
+	htUrlFlag := cli.StringFlag{Name: "url", Usage: "URL pattern (See gorilla/mux supported patterns)"}
+
+	// Resource & scale related flags (Used in env and function)
+	minCpu := cli.IntFlag{Name: "mincpu", Usage: "Minimum CPU to be assigned to pod (In millicore, minimum 1)"}
+	maxCpu := cli.IntFlag{Name: "maxcpu", Usage: "Maximum CPU to be assigned to pod (In millicore, minimum 1)"}
+	minMem := cli.IntFlag{Name: "minmemory", Usage: "Minimum memory to be assigned to pod (In megabyte)"}
+	maxMem := cli.IntFlag{Name: "maxmemory", Usage: "Maximum memory to be assigned to pod (In megabyte)"}
+	minScale := cli.IntFlag{Name: "minscale", Usage: "Minimum number of pods (Uses resource inputs to configure HPA)"}
+	maxScale := cli.IntFlag{Name: "maxscale", Usage: "Maximum number of pods (Uses resource inputs to configure HPA)"}
+	targetcpu := cli.IntFlag{Name: "targetcpu", Usage: "Target average CPU usage percentage across pods for scaling"}
+
+	// functions
+	fnNameFlag := cli.StringFlag{Name: "name", Usage: "function name"}
+	fnEnvNameFlag := cli.StringFlag{Name: "env", Usage: "environment name for function"}
+	fnCodeFlag := cli.StringFlag{Name: "code", Usage: "local path or URL for source code"}
+	fnDeployArchiveFlag := cli.StringSliceFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"}
+	fnSrcArchiveFlag := cli.StringSliceFlag{Name: "sourcearchive, src, source", Usage: "local path or URL for source archive"}
+	fnPkgNameFlag := cli.StringFlag{Name: "pkgname, pkg", Usage: "Name of the existing package (--deploy and --src and --env will be ignored), should be in the same namespace as the function"}
+	fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"}
+	fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"}
+	fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"}
+	fnLogDBTypeFlag := cli.StringFlag{Name: "dbtype", Usage: "log database type, e.g. influxdb (currently only influxdb is supported)"}
+	fnBodyFlag := cli.StringFlag{Name: "body, b", Usage: "request body"}
+	fnHeaderFlag := cli.StringSliceFlag{Name: "header, H", Usage: "request headers"}
+	fnQueryFlag := cli.StringSliceFlag{Name: "query, q", Usage: "request query parameters: -q key1=value1 -q key2=value2"}
+	fnEntryPointFlag := cli.StringFlag{Name: "entrypoint", Usage: "entry point for environment v2 to load with"}
+	fnBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "build command for builder to run with"}
+	fnSecretFlag := cli.StringFlag{Name: "secret", Usage: "function access to secret, should be present in the same namespace as the function"}
+	fnCfgMapFlag := cli.StringFlag{Name: "configmap", Usage: "function access to configmap, should be present in the same namespace as the function"}
+	fnLogCountFlag := cli.StringFlag{Name: "recordcount", Usage: "the n most recent log records"}
+	fnForceFlag := cli.BoolFlag{Name: "force", Usage: "Force update a package even if it is used by one or more functions"}
+	fnExecutorTypeFlag := cli.StringFlag{Name: "executortype", Value: types.ExecutorTypePoolmgr, Usage: "Executor type for execution; one of 'poolmgr', 'newdeploy' defaults to 'poolmgr'"}
+	fnTimeoutFlag := cli.DurationFlag{Name: "timeout, t", Value: 30 * time.Second, Usage: "The length of time to wait for the response. If set to zero or negative number, no timeout is set."}
+
+	fnSubcommands := []cli.Command{
+		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag, fnEnvNameFlag, envNamespaceFlag, specSaveFlag, fnCodeFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu, fnCfgMapFlag, fnSecretFlag}, Action: fnCreate},
+		{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag}, Action: fnGet},
+		{Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag}, Action: fnGetMeta},
+		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag, fnEnvNameFlag, envNamespaceFlag, fnCodeFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnPkgNameFlag, pkgNamespaceFlag, fnBuildCmdFlag, fnForceFlag, minCpu, maxCpu, minMem, maxMem, minScale, maxScale, fnExecutorTypeFlag, targetcpu}, Action: fnUpdate},
+		{Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag}, Action: fnDelete},
+		// TODO : for fnList, i feel like it's nice to allow --fns all, to list functions across all namespaces for cluster admins, although, this is against ns isolation.
+		// so, in the future, if we end up using kubeconfig in fission cli and enforcing rolebindings to be created for users by admins etc, we can add this option at the time.
+		{Name: "list", Usage: "List all functions in a namespace if specified, else, list functions across all namespaces", Flags: []cli.Flag{fnNamespaceFlag}, Action: fnList},
+		{Name: "logs", Usage: "Display function logs", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag, fnLogCountFlag}, Action: fnLogs},
+		{Name: "test", Usage: "Test a function", Flags: []cli.Flag{fnNameFlag, fnNamespaceFlag, fnEnvNameFlag,
+			fnCodeFlag, fnSrcArchiveFlag, htMethodFlag, fnBodyFlag, fnHeaderFlag, fnQueryFlag, fnTimeoutFlag},
+			Action: fnTest},
+	}
+
+	// httptriggers
+	htNameFlag := cli.StringFlag{Name: "name", Usage: "HTTP Trigger name"}
+	htHostFlag := cli.StringFlag{Name: "host", Usage: "FQDN of the network host for route"}
+	htIngressFlag := cli.BoolFlag{Name: "createingress", Usage: "Creates ingress with same URL, defaults to false"}
+	htFnNameFlag := cli.StringSliceFlag{Name: "function", Usage: "Name(s) of the function for this trigger. If 2 functions are supplied with this flag, traffic gets routed to them based on weights supplied with --weight flag."}
+	htFnWeightFlag := cli.IntSliceFlag{Name: "weight", Usage: "Weight for each function supplied with --function flag, in the same order. Used for canary deployment"}
+
+	htSubcommands := []cli.Command{
+
+		{Name: "create", Aliases: []string{"add"}, Usage: "Create HTTP trigger", Flags: []cli.Flag{htNameFlag, htMethodFlag, htUrlFlag, htFnNameFlag, htHostFlag, htIngressFlag, fnNamespaceFlag, specSaveFlag, htFnWeightFlag}, Action: htCreate},
+		{Name: "get", Usage: "Get HTTP trigger", Flags: []cli.Flag{htNameFlag}, Action: htGet},
+		{Name: "update", Usage: "Update HTTP trigger", Flags: []cli.Flag{htNameFlag, triggerNamespaceFlag, htFnNameFlag, htHostFlag, htIngressFlag, htFnWeightFlag}, Action: htUpdate},
+		{Name: "delete", Usage: "Delete HTTP trigger", Flags: []cli.Flag{htNameFlag, triggerNamespaceFlag}, Action: htDelete},
+		{Name: "list", Usage: "List HTTP triggers", Flags: []cli.Flag{triggerNamespaceFlag}, Action: htList},
+	}
+
+	// timetriggers
+	ttNameFlag := cli.StringFlag{Name: "name", Usage: "Time Trigger name"}
+	ttCronFlag := cli.StringFlag{Name: "cron", Usage: "Time trigger cron spec with each asterisk representing respectively second, minute, hour, the day of the month, month and day of the week. Also supports readable formats like '@every 5m', '@hourly'"}
+	ttFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"}
+	ttRoundFlag := cli.IntFlag{Name: "round", Value: 1, Usage: "Get next N rounds of invocation time"}
+	ttSubcommands := []cli.Command{
+		{Name: "create", Aliases: []string{"add"}, Usage: "Create time trigger", Flags: []cli.Flag{ttNameFlag, ttFnNameFlag, fnNamespaceFlag, ttCronFlag, specSaveFlag}, Action: ttCreate},
+		{Name: "get", Usage: "Get time trigger", Flags: []cli.Flag{triggerNamespaceFlag}, Action: ttGet},
+		{Name: "update", Usage: "Update time trigger", Flags: []cli.Flag{ttNameFlag, triggerNamespaceFlag, ttCronFlag, ttFnNameFlag}, Action: ttUpdate},
+		{Name: "delete", Usage: "Delete time trigger", Flags: []cli.Flag{ttNameFlag, triggerNamespaceFlag}, Action: ttDelete},
+		{Name: "list", Usage: "List time triggers", Flags: []cli.Flag{triggerNamespaceFlag}, Action: ttList},
+		{Name: "showschedule", Aliases: []string{"show"}, Usage: "Show schedule for cron spec", Flags: []cli.Flag{ttCronFlag, ttRoundFlag}, Action: ttTest},
+	}
+
+	// Message queue trigger
+	mqtNameFlag := cli.StringFlag{Name: "name", Usage: "Message queue Trigger name"}
+	mqtFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"}
+	mqtMQTypeFlag := cli.StringFlag{Name: "mqtype", Value: "nats-streaming", Usage: "Message queue type, e.g. nats-streaming, azure-storage-queue (optional)"}
+	mqtTopicFlag := cli.StringFlag{Name: "topic", Usage: "Message queue Topic the trigger listens on"}
+	mqtRespTopicFlag := cli.StringFlag{Name: "resptopic", Usage: "Topic that the function response is sent on (optional; response discarded if unspecified)"}
+	mqtErrorTopicFlag := cli.StringFlag{Name: "errortopic", Usage: "Topic that the function error messages are sent to (optional; errors discarded if unspecified"}
+	mqtMaxRetries := cli.IntFlag{Name: "maxretries", Value: 0, Usage: "Maximum number of times the function will be retried upon failure (optional; default is 0)"}
+	mqtMsgContentType := cli.StringFlag{Name: "contenttype, c", Value: "application/json", Usage: "Content type of messages that publish to the topic (optional)"}
+	mqtSubcommands := []cli.Command{
+		{Name: "create", Aliases: []string{"add"}, Usage: "Create Message queue trigger", Flags: []cli.Flag{mqtNameFlag, mqtFnNameFlag, fnNamespaceFlag, mqtMQTypeFlag, mqtTopicFlag, mqtRespTopicFlag, mqtErrorTopicFlag, mqtMaxRetries, mqtMsgContentType, specSaveFlag}, Action: mqtCreate},
+		{Name: "get", Usage: "Get message queue trigger", Flags: []cli.Flag{triggerNamespaceFlag}, Action: mqtGet},
+		{Name: "update", Usage: "Update message queue trigger", Flags: []cli.Flag{mqtNameFlag, triggerNamespaceFlag, mqtTopicFlag, mqtRespTopicFlag, mqtErrorTopicFlag, mqtMaxRetries, mqtFnNameFlag, mqtMsgContentType}, Action: mqtUpdate},
+		{Name: "delete", Usage: "Delete message queue trigger", Flags: []cli.Flag{mqtNameFlag, triggerNamespaceFlag}, Action: mqtDelete},
+		{Name: "list", Usage: "List message queue triggers", Flags: []cli.Flag{mqtMQTypeFlag, triggerNamespaceFlag}, Action: mqtList},
+	}
+
+	// Recorders
+	recNameFlag := cli.StringFlag{Name: "name", Usage: "Recorder name"}
+	recFnFlag := cli.StringFlag{Name: "function", Usage: "Record Function name(s): --function=fnA"}
+	recTriggersFlag := cli.StringSliceFlag{Name: "trigger", Usage: "Record Trigger name(s): --trigger=trigger1,trigger2,trigger3"}
+	//recRetentionPolFlag := cli.StringFlag{Name: "retention", Usage: "Retention policy (number of days)"}
+	//recEvictionPolFlag := cli.StringFlag{Name: "eviction", Usage: "Eviction policy (default LRU)"}
+	recEnabled := cli.BoolFlag{Name: "enable", Usage: "Enable recorder"}
+	recDisabled := cli.BoolFlag{Name: "disable", Usage: "Disable recorder"}
+	recSubcommands := []cli.Command{
+		{Name: "create", Aliases: []string{"add"}, Usage: "Create recorder", Flags: []cli.Flag{recNameFlag, recFnFlag, recTriggersFlag, specSaveFlag}, Action: recorderCreate},
+		{Name: "get", Usage: "Get recorder", Flags: []cli.Flag{recNameFlag}, Action: recorderGet},
+		{Name: "update", Usage: "Update recorder", Flags: []cli.Flag{recNameFlag, recFnFlag, recTriggersFlag, recEnabled, recDisabled}, Action: recorderUpdate},
+		{Name: "delete", Usage: "Delete recorder", Flags: []cli.Flag{recNameFlag, recorderNamespaceFlag}, Action: recorderDelete},
+		{Name: "list", Usage: "List recorders", Flags: []cli.Flag{}, Action: recorderList},
+	}
+
+	// View records
+	filterTimeFrom := cli.StringFlag{Name: "from", Usage: "Filter records by time interval; specify start of interval"}
+	filterTimeTo := cli.StringFlag{Name: "to", Usage: "Filter records by time interval; specify end of interval"}
+	filterFunction := cli.StringFlag{Name: "function", Usage: "Filter records by function"}
+	filterTrigger := cli.StringFlag{Name: "trigger", Usage: "Filter records by trigger"}
+	verbosityFlag := cli.BoolFlag{Name: "v", Usage: "Toggle verbosity -- view more detailed requests/responses"}
+	vvFlag := cli.BoolFlag{Name: "vv", Usage: "Toggle verbosity -- view raw requests/responses"}
+	recViewSubcommands := []cli.Command{
+		{Name: "view", Usage: "View existing records", Flags: []cli.Flag{filterTimeTo, filterTimeFrom, filterFunction, filterTrigger, verbosityFlag, vvFlag}, Action: recordsView},
+	}
+
+	// Replay records
+	reqIDFlag := cli.StringFlag{Name: "reqUID", Usage: "Replay a particular request by providing the reqUID (to view reqUIDs, do 'fission records view')"}
+
+	// environments
+	envNameFlag := cli.StringFlag{Name: "name", Usage: "Environment name"}
+	envPoolsizeFlag := cli.IntFlag{Name: "poolsize", Value: 3, Usage: "Size of the pool"}
+	envImageFlag := cli.StringFlag{Name: "image", Usage: "Environment image URL"}
+	envBuilderImageFlag := cli.StringFlag{Name: "builder", Usage: "Environment builder image URL (optional)"}
+	envBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for environment builder to build source package (optional)"}
+	envKeepArchiveFlag := cli.BoolFlag{Name: "keeparchive", Usage: "Keep the archive instead of extracting it into a directory (optional, defaults to false)"}
+	envExternalNetworkFlag := cli.BoolFlag{Name: "externalnetwork", Usage: "Allow environment access external network when istio feature enabled (optional, defaults to false)"}
+	envTerminationGracePeriodFlag := cli.Int64Flag{Name: "graceperiod, period", Value: 360, Usage: "The grace time (in seconds) for pod to perform connection draining before termination (optional)"}
+	envVersionFlag := cli.IntFlag{Name: "version", Value: 1, Usage: "Environment API version (1 means v1 interface)"}
+	envSubcommands := []cli.Command{
+		{Name: "create", Aliases: []string{"add"}, Usage: "Add an environment", Flags: []cli.Flag{envNameFlag, envNamespaceFlag, envPoolsizeFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag, envKeepArchiveFlag, minCpu, maxCpu, minMem, maxMem, envVersionFlag, envExternalNetworkFlag, envTerminationGracePeriodFlag, specSaveFlag}, Action: envCreate},
+		{Name: "get", Usage: "Get environment details", Flags: []cli.Flag{envNameFlag, envNamespaceFlag}, Action: envGet},
+		{Name: "update", Usage: "Update environment", Flags: []cli.Flag{envNameFlag, envNamespaceFlag, envPoolsizeFlag, envImageFlag, envBuilderImageFlag, envBuildCmdFlag, envKeepArchiveFlag, minCpu, maxCpu, minMem, maxMem, envExternalNetworkFlag, envTerminationGracePeriodFlag}, Action: envUpdate},
+		{Name: "delete", Usage: "Delete environment", Flags: []cli.Flag{envNameFlag, envNamespaceFlag}, Action: envDelete},
+		{Name: "list", Usage: "List all environments", Flags: []cli.Flag{envNamespaceFlag}, Action: envList},
+	}
+
+	// watches
+	wNameFlag := cli.StringFlag{Name: "name", Usage: "Watch name"}
+	wFnNameFlag := cli.StringFlag{Name: "function", Usage: "Function name"}
+	wNamespaceFlag := cli.StringFlag{Name: "ns", Usage: "Namespace of resource to watch"}
+	wObjTypeFlag := cli.StringFlag{Name: "type", Usage: "Type of resource to watch (Pod, Service, etc.)"}
+	wLabelsFlag := cli.StringFlag{Name: "labels", Usage: "Label selector of the form a=b,c=d"}
+	wSubCommands := []cli.Command{
+		{Name: "create", Aliases: []string{"add"}, Usage: "Create a watch", Flags: []cli.Flag{wFnNameFlag, fnNamespaceFlag, wNamespaceFlag, wObjTypeFlag, wLabelsFlag, specSaveFlag}, Action: wCreate},
+		{Name: "get", Usage: "Get details about a watch", Flags: []cli.Flag{wNameFlag, triggerNamespaceFlag}, Action: wGet},
+		// TODO add update flag when supported
+		{Name: "delete", Usage: "Delete watch", Flags: []cli.Flag{wNameFlag, triggerNamespaceFlag}, Action: wDelete},
+		{Name: "list", Usage: "List all watches", Flags: []cli.Flag{triggerNamespaceFlag}, Action: wList},
+	}
+
+	// packages
+	pkgNameFlag := cli.StringFlag{Name: "name", Usage: "Package name"}
+	pkgForceFlag := cli.BoolFlag{Name: "force, f", Usage: "Force update a package even if it is used by one or more functions"}
+	pkgEnvironmentFlag := cli.StringFlag{Name: "env", Usage: "Environment name"}
+	pkgSrcArchiveFlag := cli.StringSliceFlag{Name: "sourcearchive, src", Usage: "Local path or URL for source archive"}
+	pkgDeployArchiveFlag := cli.StringSliceFlag{Name: "deployarchive, deploy", Usage: "Local path or URL for binary archive"}
+	pkgBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for builder to run with"}
+	pkgOutputFlag := cli.StringFlag{Name: "output, o", Usage: "Output filename to save archive content"}
+	pkgOrphanFlag := cli.BoolFlag{Name: "orphan", Usage: "orphan packages that are not referenced by any function"}
+	pkgSubCommands := []cli.Command{
+		{Name: "create", Usage: "Create new package", Flags: []cli.Flag{pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag}, Action: pkgCreate},
+		{Name: "update", Usage: "Update package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgEnvironmentFlag, envNamespaceFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag, pkgForceFlag}, Action: pkgUpdate},
+		{Name: "rebuild", Usage: "Rebuild a failed package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag}, Action: pkgRebuild},
+		{Name: "getsrc", Usage: "Get source archive content", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgOutputFlag}, Action: pkgSourceGet},
+		{Name: "getdeploy", Usage: "Get deployment archive content", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgOutputFlag}, Action: pkgDeployGet},
+		{Name: "info", Usage: "Show package information", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag}, Action: pkgInfo},
+		{Name: "list", Usage: "List all packages", Flags: []cli.Flag{pkgOrphanFlag, pkgNamespaceFlag}, Action: pkgList},
+		{Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgNamespaceFlag, pkgForceFlag, pkgOrphanFlag}, Action: pkgDelete},
+	}
+
+	// upgrades, data migrations
+	upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"}
+	upgradeSubCommands := []cli.Command{
+		{Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState},
+		{Name: "restore", Usage: "Restore state dumped from a v0.1 install into a v0.2+ install", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeRestoreState},
+	}
+
+	// specs
+	specDirFlag := cli.StringFlag{Name: "specdir", Usage: "Directory to store specs, defaults to ./specs"}
+	specNameFlag := cli.StringFlag{Name: "name", Usage: "(optional) Name for the app, applied to resources as a Kubernetes annotation"}
+	specWaitFlag := cli.BoolFlag{Name: "wait", Usage: "Wait for package builds"}
+	specWatchFlag := cli.BoolFlag{Name: "watch", Usage: "Watch local files for change, and re-apply specs as necessary"}
+	specDeleteFlag := cli.BoolFlag{Name: "delete", Usage: "Allow apply to delete resources that no longer exist in the specification"}
+	specSubCommands := []cli.Command{
+		{Name: "init", Usage: "Create an initial declarative app specification", Flags: []cli.Flag{specDirFlag, specNameFlag}, Action: specInit},
+		{Name: "validate", Usage: "Validate Fission app specification", Flags: []cli.Flag{specDirFlag}, Action: specValidate},
+		{Name: "apply", Usage: "Create, update, or delete Fission resources from app specification", Flags: []cli.Flag{specDirFlag, specDeleteFlag, specWaitFlag, specWatchFlag}, Action: specApply},
+		{Name: "destroy", Usage: "Delete all Fission resources in the app specification", Flags: []cli.Flag{specDirFlag}, Action: specDestroy},
+		{Name: "helm", Usage: "Create a helm chart from the app specification", Flags: []cli.Flag{specDirFlag}, Action: specHelm, Hidden: true},
+	}
+
+	// support
+	supportOutputFlag := cli.StringFlag{Name: "output, o", Value: support.DEFAULT_OUTPUT_DIR, Usage: "Output directory to save dump archive/files"}
+	supportNoZipFlag := cli.BoolFlag{Name: "nozip", Usage: "Save dump information into multiple files instead of single zip file"}
+	supportSubCommands := []cli.Command{
+		{Name: "dump", Usage: "Collect & dump all necessary for troubleshooting", Flags: []cli.Flag{supportOutputFlag, supportNoZipFlag}, Action: support.DumpInfo},
+	}
+
+	// canary configs
+	canaryConfigNameFlag := cli.StringFlag{Name: "name", Usage: "Name for the canary config"}
+	triggerNameFlag := cli.StringFlag{Name: "httptrigger", Usage: "Http trigger that this config references"}
+	newFunc := cli.StringFlag{Name: "newfunction", Usage: "New version of the function"}
+	oldFunc := cli.StringFlag{Name: "oldfunction", Usage: "Old stable version of the function"}
+	weightIncrementFlag := cli.IntFlag{Name: "increment-step", Value: 20, Usage: "Weight increment step for function"}
+	incrementIntervalFlag := cli.StringFlag{Name: "increment-interval", Value: "2m", Usage: "Weight increment interval, string representation of time.Duration, ex : 1m, 2h, 2d"}
+	failureThresholdFlag := cli.IntFlag{Name: "failure-threshold", Value: 10, Usage: "Threshold in percentage beyond which the new version of the function is considered unstable"}
+	canarySubCommands := []cli.Command{
+		{Name: "create", Usage: "Create a canary config", Flags: []cli.Flag{canaryConfigNameFlag, triggerNameFlag, newFunc, oldFunc, fnNamespaceFlag, weightIncrementFlag, incrementIntervalFlag, failureThresholdFlag}, Action: canaryConfigCreate},
+		{Name: "get", Usage: "View parameters in a canary config", Flags: []cli.Flag{canaryConfigNameFlag, canaryNamespaceFlag}, Action: canaryConfigGet},
+		{Name: "update", Usage: "Update parameters of a canary config", Flags: []cli.Flag{canaryConfigNameFlag, canaryNamespaceFlag, incrementIntervalFlag, weightIncrementFlag, failureThresholdFlag}, Action: canaryConfigUpdate},
+		{Name: "delete", Usage: "Delete a canary config", Flags: []cli.Flag{canaryConfigNameFlag, canaryNamespaceFlag}, Action: canaryConfigDelete},
+		{Name: "list", Usage: "List all canary configs in a namespace", Flags: []cli.Flag{canaryNamespaceFlag}, Action: canaryConfigList},
+	}
+
+	app.Commands = []cli.Command{
+		{Name: "function", Aliases: []string{"fn"}, Usage: "Create, update and manage functions", Subcommands: fnSubcommands},
+		{Name: "httptrigger", Aliases: []string{"ht", "route"}, Usage: "Manage HTTP triggers (routes) for functions", Subcommands: htSubcommands},
+		{Name: "timetrigger", Aliases: []string{"tt", "timer"}, Usage: "Manage Time triggers (timers) for functions", Subcommands: ttSubcommands},
+		{Name: "mqtrigger", Aliases: []string{"mqt", "messagequeue"}, Usage: "Manage message queue triggers for functions", Subcommands: mqtSubcommands},
+		{Name: "recorder", Usage: "Manage recorders for functions", Subcommands: recSubcommands, Hidden: true},
+		{Name: "records", Usage: "View records with optional filters", Subcommands: recViewSubcommands, Hidden: true},
+		{Name: "replay", Usage: "Replay records", Flags: []cli.Flag{reqIDFlag}, Action: replay},
+		{Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands},
+		{Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands},
+		{Name: "package", Aliases: []string{"pkg"}, Usage: "Manage packages", Subcommands: pkgSubCommands},
+		{Name: "spec", Aliases: []string{"specs"}, Usage: "Manage a declarative app specification", Subcommands: specSubCommands},
+		{Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands},
+		{Name: "support", Usage: "Collect an archive of diagnostic information for support", Subcommands: supportSubCommands},
+		cmdPlugin,
+		{Name: "canary-config", Aliases: []string{}, Usage: "Create, Update and manage Canary Configs", Subcommands: canarySubCommands},
+	}
+
+	app.Before = cliHook
+	app.Action = handleNoCommand
+	return app
+}
+
+func handleNoCommand(ctx *cli.Context) error {
+	if ctx.GlobalBool("version") {
+		versionPrinter(ctx)
+		return nil
+	}
+	if ctx.GlobalBool("plugin") {
+		bs, err := json.Marshal(plugin.Metadata{
+			Version: info.Version,
+			Usage:   ctx.App.Usage,
+		})
+		if err != nil {
+			log.Fatal(fmt.Sprintf("Failed to marshal plugin metadata to JSON: %v", err))
+		}
+		fmt.Println(string(bs))
+		return nil
+	}
+	if len(ctx.Args()) > 0 {
+		handleCommandNotFound(ctx, ctx.Args().First())
+		return nil
+	}
+
+	return cli.ShowAppHelp(ctx)
+}
+
+func handleCommandNotFound(ctx *cli.Context, subCommand string) {
+	pmd, err := plugin.Find(subCommand)
+	if err != nil {
+		switch err {
+		case plugin.ErrPluginNotFound:
+			url, ok := plugin.SearchRegistries(subCommand)
+			if !ok {
+				log.Fatal("No help topic for '" + subCommand + "'")
+			}
+			log.Fatal(fmt.Sprintf(`Command '%v' is not installed.
+It is available to download at '%v'.
+
+To install it for your local Fission CLI:
+1. Download the plugin binary for your OS from the URL
+2. Ensure that the plugin binary is executable: chmod +x <binary>
+2. Add the plugin binary to your $PATH: mv <binary> /usr/local/bin/fission-%v`, subCommand, url, subCommand))
+		default:
+			log.Fatal("Error occurred when invoking " + subCommand + ": " + err.Error())
+		}
+		os.Exit(1)
+	}
+
+	// Rebuild global arguments string (urfave/cli does not have an option to get the raw input of the global flags)
+	var globalArgs []string
+	for _, globalFlagName := range ctx.GlobalFlagNames() {
+		if globalFlagName == "plugin" {
+			continue
+		}
+		val := fmt.Sprintf("%v", ctx.GlobalGeneric(globalFlagName))
+		if len(val) > 0 {
+			globalArgs = append(globalArgs, fmt.Sprintf("--%v", globalFlagName), val)
+		}
+	}
+	args := append(globalArgs, ctx.Args().Tail()...)
+
+	err = plugin.Exec(pmd, args)
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func versionPrinter(_ *cli.Context) {
+	client := util.GetApiClient(util.GetServerUrl())
+	ver := util.GetVersion(client)
+	fmt.Print(string(ver))
+}
+
+func flagValueParser(args []string) error {
+	// all input value for flags are properly set
+	if len(args) == 0 {
+		return nil
+	}
+
+	var flagIndexes []int
+	var errorFlags []string
+
+	// find out all flag indexes
+	for i, v := range args {
+		// support both flags with "--" and "-"
+		if strings.HasPrefix(v, "-") {
+			flagIndexes = append(flagIndexes, i)
+		}
+	}
+
+	// add total length of args to indicate the end of args
+	flagIndexes = append(flagIndexes, len(args))
+
+	for i := 0; i < len(flagIndexes)-1; i++ {
+		// if the difference between the flag index i and i+1
+		// is bigger then 2 means that CLI receives extra arguments
+		// for one flag. For example,
+		// 1. fission fn create --name e1 --code examples/nodejs/* --env nodejs ...
+		//    The wildcard will be extracted to multiple files and cause the difference between `--code` and `--env` large than 2.
+		// 2. fission fn create --spec --name e1 ...
+		//    The difference between --spec and --name is 1.
+		if flagIndexes[i+1]-flagIndexes[i] > 2 {
+			index := flagIndexes[i]
+			errorFlags = append(errorFlags, args[index])
+		}
+	}
+
+	if len(errorFlags) > 0 {
+		e := fmt.Sprintf("Unable to parse flags: %v\nThe argument should have only one input value. Please quote the input value if it contains wildcard characters(*).", strings.Join(errorFlags[:], ", "))
+		return errors.New(e)
+	}
+
+	return nil
+}
+
+var helpTemplate = `NAME:
+   {{.Name}}{{if .Usage}} - {{.Usage}}{{end}}
+
+USAGE:
+   {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .VisibleFlags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}{{if .Version}}{{if not .HideVersion}}
+
+VERSION:
+   {{.Version}}{{end}}{{end}}{{if .Description}}
+
+DESCRIPTION:
+   {{.Description}}{{end}}{{if .VisibleCommands}}
+
+COMMANDS:{{range .VisibleCategories}}{{if .Name}}
+   {{.Name}}:{{end}}{{range .VisibleCommands}}
+     {{join .Names ", "}}{{"\t"}}{{.Usage}}{{end}}{{end}}{{end}}{{if .VisibleFlags}}
+
+PLUGIN COMMANDS:{{ range $name, $usage := ExtraInfo }}
+     {{$name}}{{"\t"}}{{$usage}}{{end}}
+
+GLOBAL OPTIONS:
+   {{range $index, $option := .VisibleFlags}}{{if $index}}
+   {{end}}{{$option}}{{end}}{{end}}
+`

pkg/fission-cli/mqtrigger.go

@@ -0,0 +1,235 @@
+/*
+Copyrigtt 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    tttp://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+
+	"github.com/satori/go.uuid"
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/util"
+	"github.com/fission/fission/pkg/types"
+)
+
+func mqtCreate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	mqtName := c.String("name")
+	if len(mqtName) == 0 {
+		mqtName = uuid.NewV4().String()
+	}
+	fnName := c.String("function")
+	if len(fnName) == 0 {
+		log.Fatal("Need a function name to create a trigger, use --function")
+	}
+	fnNamespace := c.String("fnNamespace")
+
+	var mqType fv1.MessageQueueType
+	switch c.String("mqtype") {
+	case "":
+		mqType = types.MessageQueueTypeNats
+	case types.MessageQueueTypeNats:
+		mqType = types.MessageQueueTypeNats
+	case types.MessageQueueTypeASQ:
+		mqType = types.MessageQueueTypeASQ
+	case types.MessageQueueTypeKafka:
+		mqType = types.MessageQueueTypeKafka
+
+	default:
+		log.Fatal("Unknown message queue type, currently only \"nats-streaming, azure-storage-queue, kafka \" is supported")
+
+	}
+
+	// TODO: check topic availability
+	topic := c.String("topic")
+	if len(topic) == 0 {
+		log.Fatal("Topic cannot be empty")
+	}
+	respTopic := c.String("resptopic")
+
+	if topic == respTopic {
+		// TODO maybe this should just be a warning, perhaps
+		// allow it behind a --force flag
+		log.Fatal("Listen topic should not equal to response topic")
+	}
+
+	errorTopic := c.String("errortopic")
+
+	maxRetries := c.Int("maxretries")
+
+	if maxRetries < 0 {
+		log.Fatal("Maximum number of retries must be a natural number, default is 0")
+	}
+
+	contentType := c.String("contenttype")
+	if len(contentType) == 0 {
+		contentType = "application/json"
+	}
+
+	checkMQTopicAvailability(mqType, topic, respTopic)
+
+	mqt := &fv1.MessageQueueTrigger{
+		Metadata: metav1.ObjectMeta{
+			Name:      mqtName,
+			Namespace: fnNamespace,
+		},
+		Spec: fv1.MessageQueueTriggerSpec{
+			FunctionReference: fv1.FunctionReference{
+				Type: types.FunctionReferenceTypeFunctionName,
+				Name: fnName,
+			},
+			MessageQueueType: mqType,
+			Topic:            topic,
+			ResponseTopic:    respTopic,
+			ErrorTopic:       errorTopic,
+			MaxRetries:       maxRetries,
+			ContentType:      contentType,
+		},
+	}
+
+	// if we're writing a spec, don't call the API
+	if c.Bool("spec") {
+		specFile := fmt.Sprintf("mqtrigger-%v.yaml", mqtName)
+		err := specSave(*mqt, specFile)
+		util.CheckErr(err, "create message queue trigger spec")
+		return nil
+	}
+
+	_, err := client.MessageQueueTriggerCreate(mqt)
+	util.CheckErr(err, "create message queue trigger")
+
+	fmt.Printf("trigger '%s' created\n", mqtName)
+	return err
+}
+
+func mqtGet(c *cli.Context) error {
+	return nil
+}
+
+func mqtUpdate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+	mqtName := c.String("name")
+	if len(mqtName) == 0 {
+		log.Fatal("Need name of trigger, use --name")
+	}
+	mqtNs := c.String("triggerns")
+
+	topic := c.String("topic")
+	respTopic := c.String("resptopic")
+	errorTopic := c.String("errortopic")
+	maxRetries := c.Int("maxretries")
+	fnName := c.String("function")
+	contentType := c.String("contenttype")
+
+	mqt, err := client.MessageQueueTriggerGet(&metav1.ObjectMeta{
+		Name:      mqtName,
+		Namespace: mqtNs,
+	})
+	util.CheckErr(err, "get Time trigger")
+
+	// TODO : Find out if we can make a call to checkIfFunctionExists, in the same ns more importantly.
+
+	checkMQTopicAvailability(mqt.Spec.MessageQueueType, topic, respTopic)
+
+	updated := false
+	if len(topic) > 0 {
+		mqt.Spec.Topic = topic
+		updated = true
+	}
+	if len(respTopic) > 0 {
+		mqt.Spec.ResponseTopic = respTopic
+		updated = true
+	}
+	if len(errorTopic) > 0 {
+		mqt.Spec.ErrorTopic = errorTopic
+		updated = true
+	}
+	if maxRetries > -1 {
+		mqt.Spec.MaxRetries = maxRetries
+		updated = true
+	}
+	if len(fnName) > 0 {
+		mqt.Spec.FunctionReference.Name = fnName
+		updated = true
+	}
+	if len(contentType) > 0 {
+		mqt.Spec.ContentType = contentType
+		updated = true
+	}
+
+	if !updated {
+		log.Fatal("Nothing to update. Use --topic, --resptopic, --errortopic, --maxretries or --function.")
+	}
+
+	_, err = client.MessageQueueTriggerUpdate(mqt)
+	util.CheckErr(err, "update Time trigger")
+
+	fmt.Printf("trigger '%v' updated\n", mqtName)
+	return nil
+}
+
+func mqtDelete(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+	mqtName := c.String("name")
+	if len(mqtName) == 0 {
+		log.Fatal("Need name of trigger to delete, use --name")
+	}
+	mqtNs := c.String("triggerns")
+
+	err := client.MessageQueueTriggerDelete(&metav1.ObjectMeta{
+		Name:      mqtName,
+		Namespace: mqtNs,
+	})
+	util.CheckErr(err, "delete trigger")
+
+	fmt.Printf("trigger '%v' deleted\n", mqtName)
+	return nil
+}
+
+func mqtList(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+	mqtNs := c.String("triggerns")
+
+	mqts, err := client.MessageQueueTriggerList(c.String("mqtype"), mqtNs)
+	util.CheckErr(err, "list message queue triggers")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+		"NAME", "FUNCTION_NAME", "MESSAGE_QUEUE_TYPE", "TOPIC", "RESPONSE_TOPIC", "ERROR_TOPIC", "MAX_RETRIES", "PUB_MSG_CONTENT_TYPE")
+	for _, mqt := range mqts {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
+			mqt.Metadata.Name, mqt.Spec.FunctionReference.Name, mqt.Spec.MessageQueueType, mqt.Spec.Topic, mqt.Spec.ResponseTopic, mqt.Spec.ErrorTopic, mqt.Spec.MaxRetries, mqt.Spec.ContentType)
+	}
+	w.Flush()
+
+	return nil
+}
+
+func checkMQTopicAvailability(mqType fv1.MessageQueueType, topics ...string) {
+	for _, t := range topics {
+		if len(t) > 0 && !fv1.IsTopicValid(mqType, t) {
+			log.Fatal(fmt.Sprintf("Invalid topic for %s: %s", mqType, t))
+		}
+	}
+}

pkg/fission-cli/package.go

@@ -0,0 +1,771 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/dchest/uniuri"
+	"github.com/fission/fission/pkg/utils"
+	"github.com/hashicorp/go-multierror"
+	"github.com/mholt/archiver"
+	"github.com/pkg/errors"
+	"github.com/satori/go.uuid"
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/controller/client"
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/util"
+	storageSvcClient "github.com/fission/fission/pkg/storagesvc/client"
+	"github.com/fission/fission/pkg/types"
+)
+
+func getFunctionsByPackage(client *client.Client, pkgName, pkgNamespace string) ([]fv1.Function, error) {
+	fnList, err := client.FunctionList(pkgNamespace)
+	if err != nil {
+		return nil, err
+	}
+	fns := []fv1.Function{}
+	for _, fn := range fnList {
+		if fn.Spec.Package.PackageRef.Name == pkgName {
+			fns = append(fns, fn)
+		}
+	}
+	return fns, nil
+}
+
+// downloadStoragesvcURL downloads and return archive content with given storage service url
+func downloadStoragesvcURL(client *client.Client, fileUrl string) io.ReadCloser {
+	u, err := url.ParseRequestURI(fileUrl)
+	if err != nil {
+		return nil
+	}
+
+	// replace in-cluster storage service host with controller server url
+	fileDownloadUrl := strings.TrimSuffix(client.Url, "/") + "/proxy/storage/" + u.RequestURI()
+	reader, err := downloadURL(fileDownloadUrl)
+
+	util.CheckErr(err, fmt.Sprintf("download from storage service url: %v", fileUrl))
+	return reader
+}
+
+func pkgCreate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	pkgNamespace := c.String("pkgNamespace")
+	envName := c.String("env")
+	if len(envName) == 0 {
+		log.Fatal("Need --env argument.")
+	}
+	envNamespace := c.String("envNamespace")
+	srcArchiveFiles := c.StringSlice("src")
+	deployArchiveFiles := c.StringSlice("deploy")
+	buildcmd := c.String("buildcmd")
+
+	if len(srcArchiveFiles) == 0 && len(deployArchiveFiles) == 0 {
+		log.Fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.")
+	}
+
+	createPackage(client, pkgNamespace, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, "", "", false)
+
+	return nil
+}
+
+func pkgUpdate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		log.Fatal("Need --name argument.")
+	}
+	pkgNamespace := c.String("pkgNamespace")
+
+	force := c.Bool("f")
+	envName := c.String("env")
+	envNamespace := c.String("envNamespace")
+	srcArchiveFiles := c.StringSlice("src")
+	deployArchiveFiles := c.StringSlice("deploy")
+	buildcmd := c.String("buildcmd")
+
+	if len(srcArchiveFiles) > 0 && len(deployArchiveFiles) > 0 {
+		log.Fatal("Need either of --src or --deploy and not both arguments.")
+	}
+
+	if len(srcArchiveFiles) == 0 && len(deployArchiveFiles) == 0 &&
+		len(envName) == 0 && len(buildcmd) == 0 {
+		log.Fatal("Need --env or --src or --deploy or --buildcmd argument.")
+	}
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: pkgNamespace,
+		Name:      pkgName,
+	})
+	util.CheckErr(err, "get package")
+
+	// if the new env specified is the same as the old one, no need to update package
+	// same is true for all update parameters, but, for now, we dont check all of them - because, its ok to
+	// re-write the object with same old values, we just end up getting a new resource version for the object.
+	if len(envName) > 0 && envName == pkg.Spec.Environment.Name {
+		envName = ""
+	}
+
+	if envNamespace == pkg.Spec.Environment.Namespace {
+		envNamespace = ""
+	}
+
+	fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name, pkg.Metadata.Namespace)
+	util.CheckErr(err, "get function list")
+
+	if !force && len(fnList) > 1 {
+		log.Fatal("Package is used by multiple functions, use --force to force update")
+	}
+
+	newPkgMeta, err := updatePackage(client, pkg,
+		envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, false, false)
+	if err != nil {
+		util.CheckErr(err, "update package")
+	}
+
+	// update resource version of package reference of functions that shared the same package
+	for _, fn := range fnList {
+		fn.Spec.Package.PackageRef.ResourceVersion = newPkgMeta.ResourceVersion
+		_, err := client.FunctionUpdate(&fn)
+		util.CheckErr(err, "update function")
+	}
+
+	fmt.Printf("Package '%v' updated\n", newPkgMeta.GetName())
+
+	return nil
+}
+
+func updatePackage(client *client.Client, pkg *fv1.Package, envName, envNamespace string,
+	srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, forceRebuild bool, noZip bool) (*metav1.ObjectMeta, error) {
+
+	var srcArchiveMetadata, deployArchiveMetadata *fv1.Archive
+	needToBuild := false
+
+	if len(envName) > 0 {
+		pkg.Spec.Environment.Name = envName
+		needToBuild = true
+	}
+
+	if len(envNamespace) > 0 {
+		pkg.Spec.Environment.Namespace = envNamespace
+		needToBuild = true
+	}
+
+	if len(buildcmd) > 0 {
+		pkg.Spec.BuildCommand = buildcmd
+		needToBuild = true
+	}
+
+	if len(srcArchiveFiles) > 0 {
+		srcArchiveMetadata = createArchive(client, srcArchiveFiles, false, "", "")
+		pkg.Spec.Source = *srcArchiveMetadata
+		needToBuild = true
+	}
+
+	if len(deployArchiveFiles) > 0 {
+		deployArchiveMetadata = createArchive(client, deployArchiveFiles, noZip, "", "")
+		pkg.Spec.Deployment = *deployArchiveMetadata
+		// Users may update the env, envNS and deploy archive at the same time,
+		// but without the source archive. In this case, we should set needToBuild to false
+		needToBuild = false
+	}
+
+	// Set package as pending status when needToBuild is true
+	if needToBuild || forceRebuild {
+		// change into pending state to trigger package build
+		pkg.Status = fv1.PackageStatus{
+			BuildStatus: fv1.BuildStatusPending,
+		}
+	}
+
+	newPkgMeta, err := client.PackageUpdate(pkg)
+	util.CheckErr(err, "update package")
+
+	return newPkgMeta, err
+}
+
+func pkgSourceGet(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		log.Fatal("Need name of package, use --name")
+	}
+	pkgNamespace := c.String("pkgNamespace")
+
+	output := c.String("output")
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: pkgNamespace,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	var reader io.Reader
+
+	if pkg.Spec.Source.Type == fv1.ArchiveTypeLiteral {
+		reader = bytes.NewReader(pkg.Spec.Source.Literal)
+	} else if pkg.Spec.Source.Type == fv1.ArchiveTypeUrl {
+		readCloser := downloadStoragesvcURL(client, pkg.Spec.Source.URL)
+		defer readCloser.Close()
+		reader = readCloser
+	}
+
+	if len(output) > 0 {
+		return writeArchiveToFile(output, reader)
+	} else {
+		_, err := io.Copy(os.Stdout, reader)
+		return err
+	}
+}
+
+func pkgDeployGet(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		log.Fatal("Need name of package, use --name")
+	}
+	pkgNamespace := c.String("pkgNamespace")
+
+	output := c.String("output")
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: pkgNamespace,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	var reader io.Reader
+
+	if pkg.Spec.Deployment.Type == fv1.ArchiveTypeLiteral {
+		reader = bytes.NewReader(pkg.Spec.Deployment.Literal)
+	} else if pkg.Spec.Deployment.Type == fv1.ArchiveTypeUrl {
+		readCloser := downloadStoragesvcURL(client, pkg.Spec.Deployment.URL)
+		defer readCloser.Close()
+		reader = readCloser
+	}
+
+	if len(output) > 0 {
+		return writeArchiveToFile(output, reader)
+	} else {
+		_, err := io.Copy(os.Stdout, reader)
+		return err
+	}
+}
+
+func pkgInfo(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		log.Fatal("Need name of package, use --name")
+	}
+	pkgNamespace := c.String("pkgNamespace")
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: pkgNamespace,
+		Name:      pkgName,
+	})
+	if err != nil {
+		util.CheckErr(err, fmt.Sprintf("find package %s", pkgName))
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\n", "Name:", pkg.Metadata.Name)
+	fmt.Fprintf(w, "%v\t%v\n", "Environment:", pkg.Spec.Environment.Name)
+	fmt.Fprintf(w, "%v\t%v\n", "Status:", pkg.Status.BuildStatus)
+	fmt.Fprintf(w, "%v\n%v", "Build Logs:", pkg.Status.BuildLog)
+	w.Flush()
+
+	return nil
+}
+
+func pkgList(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+	// option for the user to list all orphan packages (not referenced by any function)
+	listOrphans := c.Bool("orphan")
+	pkgNamespace := c.String("pkgNamespace")
+
+	pkgList, err := client.PackageList(pkgNamespace)
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "BUILD_STATUS", "ENV")
+	if listOrphans {
+		for _, pkg := range pkgList {
+			fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name, pkg.Metadata.Namespace)
+			util.CheckErr(err, fmt.Sprintf("get functions sharing package %s", pkg.Metadata.Name))
+			if len(fnList) == 0 {
+				fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name, pkg.Status.BuildStatus, pkg.Spec.Environment.Name)
+			}
+		}
+	} else {
+		for _, pkg := range pkgList {
+			fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name,
+				pkg.Status.BuildStatus, pkg.Spec.Environment.Name)
+		}
+	}
+
+	w.Flush()
+
+	return nil
+}
+
+func deleteOrphanPkgs(client *client.Client, pkgNamespace string) error {
+	pkgList, err := client.PackageList(pkgNamespace)
+	if err != nil {
+		return err
+	}
+
+	// range through all packages and find out the ones not referenced by any function
+	for _, pkg := range pkgList {
+		fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name, pkgNamespace)
+		util.CheckErr(err, fmt.Sprintf("get functions sharing package %s", pkg.Metadata.Name))
+		if len(fnList) == 0 {
+			err = deletePackage(client, pkg.Metadata.Name, pkgNamespace)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func deletePackage(client *client.Client, pkgName string, pkgNamespace string) error {
+	return client.PackageDelete(&metav1.ObjectMeta{
+		Namespace: pkgNamespace,
+		Name:      pkgName,
+	})
+}
+
+func pkgDelete(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	pkgNamespace := c.String("pkgNamespace")
+	deleteOrphans := c.Bool("orphan")
+
+	if len(pkgName) == 0 && !deleteOrphans {
+		fmt.Println("Need --name argument or --orphan flag.")
+		return nil
+	}
+	if len(pkgName) != 0 && deleteOrphans {
+		fmt.Println("Need either --name argument or --orphan flag")
+		return nil
+	}
+
+	if len(pkgName) != 0 {
+		force := c.Bool("f")
+
+		_, err := client.PackageGet(&metav1.ObjectMeta{
+			Namespace: pkgNamespace,
+			Name:      pkgName,
+		})
+		util.CheckErr(err, "find package")
+
+		fnList, err := getFunctionsByPackage(client, pkgName, pkgNamespace)
+
+		if !force && len(fnList) > 0 {
+			log.Fatal("Package is used by at least one function, use -f to force delete")
+		}
+
+		err = deletePackage(client, pkgName, pkgNamespace)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Package '%v' deleted\n", pkgName)
+	} else {
+		err := deleteOrphanPkgs(client, pkgNamespace)
+		util.CheckErr(err, "error deleting orphan packages")
+		fmt.Println("Orphan packages deleted")
+	}
+
+	return nil
+}
+
+func pkgRebuild(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		log.Fatal("Need name of package, use --name")
+	}
+	pkgNamespace := c.String("pkgNamespace")
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Name:      pkgName,
+		Namespace: pkgNamespace,
+	})
+	util.CheckErr(err, "find package")
+
+	if pkg.Status.BuildStatus != fv1.BuildStatusFailed {
+		log.Fatal(fmt.Sprintf("Package %v is not in %v state.",
+			pkg.Metadata.Name, fv1.BuildStatusFailed))
+	}
+
+	_, err = updatePackage(client, pkg, "", "", nil, nil, "", true, false)
+	util.CheckErr(err, "update package")
+
+	fmt.Printf("Retrying build for pkg %v. Use \"fission pkg info --name %v\" to view status.\n", pkg.Metadata.Name, pkg.Metadata.Name)
+
+	return nil
+}
+
+func fileSize(filePath string) int64 {
+	info, err := os.Stat(filePath)
+	util.CheckErr(err, fmt.Sprintf("stat %v", filePath))
+	return info.Size()
+}
+
+func fileChecksum(fileName string) (*fv1.Checksum, error) {
+	f, err := os.Open(fileName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file %v: %v", fileName, err)
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	_, err = io.Copy(h, f)
+	if err != nil {
+		return nil, fmt.Errorf("failed to calculate checksum for %v", fileName)
+	}
+
+	return &fv1.Checksum{
+		Type: fv1.ChecksumTypeSHA256,
+		Sum:  hex.EncodeToString(h.Sum(nil)),
+	}, nil
+}
+
+// Return a fv1.Archive made from an archive .  If specFile, then
+// create an archive upload spec in the specs directory; otherwise
+// upload the archive using client.  noZip avoids zipping the
+// includeFiles, but is ignored if there's more than one includeFile.
+func createArchive(client *client.Client, includeFiles []string, noZip bool, specDir string, specFile string) *fv1.Archive {
+
+	var errs *multierror.Error
+
+	// check files existence
+	for _, path := range includeFiles {
+		// ignore http files
+		if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
+			continue
+		}
+
+		// Get files from inputs as number of files decide next steps
+		files, err := utils.FindAllGlobs([]string{path})
+		if err != nil {
+			util.CheckErr(err, "finding all globs")
+		}
+
+		if len(files) == 0 {
+			errs = multierror.Append(errs, errors.New(fmt.Sprintf("Error finding any files with path \"%v\"", path)))
+		}
+	}
+
+	if errs.ErrorOrNil() != nil {
+		log.Fatal(errs.Error())
+	}
+
+	if len(specFile) > 0 {
+		// create an ArchiveUploadSpec and reference it from the archive
+		aus := &ArchiveUploadSpec{
+			Name:         archiveName("", includeFiles),
+			IncludeGlobs: includeFiles,
+		}
+
+		// check if this AUS exists in the specs; if so, don't create a new one
+		fr, err := readSpecs(specDir)
+		util.CheckErr(err, "read specs")
+		if m := fr.specExists(aus, false, true); m != nil {
+			fmt.Printf("Re-using previously created archive %v\n", m.Name)
+			aus.Name = m.Name
+		} else {
+			// save the uploadspec
+			err := specSave(*aus, specFile)
+			util.CheckErr(err, fmt.Sprintf("write spec file %v", specFile))
+		}
+
+		// create the archive object
+		ar := &fv1.Archive{
+			Type: fv1.ArchiveTypeUrl,
+			URL:  fmt.Sprintf("%v%v", ARCHIVE_URL_PREFIX, aus.Name),
+		}
+		return ar
+	}
+
+	archivePath := makeArchiveFileIfNeeded("", includeFiles, noZip)
+
+	ctx := context.Background()
+	return uploadArchive(ctx, client, archivePath)
+}
+
+func uploadArchive(ctx context.Context, client *client.Client, fileName string) *fv1.Archive {
+	var archive fv1.Archive
+
+	// If filename is a URL, download it first
+	if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") {
+		fileName = downloadToTempFile(fileName)
+	}
+
+	if fileSize(fileName) < types.ArchiveLiteralSizeLimit {
+		archive.Type = fv1.ArchiveTypeLiteral
+		archive.Literal = getContents(fileName)
+	} else {
+		u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage"
+		ssClient := storageSvcClient.MakeClient(u)
+
+		// TODO add a progress bar
+		id, err := ssClient.Upload(ctx, fileName, nil)
+		util.CheckErr(err, fmt.Sprintf("upload file %v", fileName))
+
+		storageSvc, err := client.GetSvcURL("application=fission-storage")
+		storageSvcURL := "http://" + storageSvc
+		util.CheckErr(err, "get fission storage service name")
+
+		// We make a new client with actual URL of Storage service so that the URL is not
+		// pointing to 127.0.0.1 i.e. proxy. DON'T reuse previous ssClient
+		pkgClient := storageSvcClient.MakeClient(storageSvcURL)
+		archiveURL := pkgClient.GetUrl(id)
+
+		archive.Type = fv1.ArchiveTypeUrl
+		archive.URL = archiveURL
+
+		csum, err := fileChecksum(fileName)
+		util.CheckErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
+
+		archive.Checksum = *csum
+	}
+	return &archive
+}
+
+func createPackage(client *client.Client, pkgNamespace string, envName string, envNamespace string, srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool) *metav1.ObjectMeta {
+	pkgSpec := fv1.PackageSpec{
+		Environment: fv1.EnvironmentReference{
+			Namespace: envNamespace,
+			Name:      envName,
+		},
+	}
+	var pkgStatus fv1.BuildStatus = fv1.BuildStatusSucceeded
+
+	var pkgName string
+	if len(deployArchiveFiles) > 0 {
+		if len(specFile) > 0 { // we should do this in all cases, i think
+			pkgStatus = fv1.BuildStatusNone
+		}
+		pkgSpec.Deployment = *createArchive(client, deployArchiveFiles, noZip, specDir, specFile)
+		pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(deployArchiveFiles[0]), uniuri.NewLen(4)))
+	}
+	if len(srcArchiveFiles) > 0 {
+		pkgSpec.Source = *createArchive(client, srcArchiveFiles, false, specDir, specFile)
+		pkgStatus = fv1.BuildStatusPending // set package build status to pending
+		pkgName = util.KubifyName(fmt.Sprintf("%v-%v", path.Base(srcArchiveFiles[0]), uniuri.NewLen(4)))
+	}
+
+	if len(buildcmd) > 0 {
+		pkgSpec.BuildCommand = buildcmd
+	}
+
+	if len(pkgName) == 0 {
+		pkgName = strings.ToLower(uuid.NewV4().String())
+	}
+	pkg := &fv1.Package{
+		Metadata: metav1.ObjectMeta{
+			Name:      pkgName,
+			Namespace: pkgNamespace,
+		},
+		Spec: pkgSpec,
+		Status: fv1.PackageStatus{
+			BuildStatus: pkgStatus,
+		},
+	}
+
+	if len(specFile) > 0 {
+		// if a package sith the same spec exists, don't create a new spec file
+		fr, err := readSpecs(getSpecDir(nil))
+		util.CheckErr(err, "read specs")
+		if m := fr.specExists(pkg, false, true); m != nil {
+			fmt.Printf("Re-using previously created package %v\n", m.Name)
+			return m
+		}
+
+		err = specSave(*pkg, specFile)
+		util.CheckErr(err, "save package spec")
+		return &pkg.Metadata
+	} else {
+		pkgMetadata, err := client.PackageCreate(pkg)
+		util.CheckErr(err, "create package")
+		fmt.Printf("Package '%v' created\n", pkgMetadata.GetName())
+		return pkgMetadata
+	}
+}
+
+func getContents(filePath string) []byte {
+	var code []byte
+	var err error
+
+	code, err = ioutil.ReadFile(filePath)
+	util.CheckErr(err, fmt.Sprintf("read %v", filePath))
+	return code
+}
+
+func writeArchiveToFile(fileName string, reader io.Reader) error {
+	tmpDir, err := utils.GetTempDir()
+	if err != nil {
+		return err
+	}
+
+	path := filepath.Join(tmpDir, fileName+".tmp")
+	w, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(w, reader)
+	if err != nil {
+		return err
+	}
+	err = os.Chmod(path, 0644)
+	if err != nil {
+		return err
+	}
+
+	err = os.Rename(path, fileName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// downloadToTempFile fetches archive file from arbitrary url
+// and write it to temp file for further usage
+func downloadToTempFile(fileUrl string) string {
+	reader, err := downloadURL(fileUrl)
+	defer reader.Close()
+	util.CheckErr(err, fmt.Sprintf("download from url: %v", fileUrl))
+
+	tmpDir, err := utils.GetTempDir()
+	util.CheckErr(err, "create temp directory")
+
+	tmpFilename := uuid.NewV4().String()
+	destination := filepath.Join(tmpDir, tmpFilename)
+	err = os.Mkdir(tmpDir, 0744)
+	util.CheckErr(err, "create temp directory")
+
+	err = writeArchiveToFile(destination, reader)
+	util.CheckErr(err, "write archive to file")
+
+	return destination
+}
+
+// downloadURL downloads file from given url
+func downloadURL(fileUrl string) (io.ReadCloser, error) {
+	resp, err := http.Get(fileUrl)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode)
+	}
+	return resp.Body, nil
+}
+
+// Create an archive from the given list of input files, unless that
+// list has only one item and that item is either a zip file or a URL.
+//
+// If the inputs have only one file and noZip is true, the file is
+// returned as-is with no zipping.  (This is used for compatibility
+// with v1 envs.)  noZip is IGNORED if there is more than one input
+// file.
+func makeArchiveFileIfNeeded(archiveNameHint string, archiveInput []string, noZip bool) string {
+
+	// Unique name for the archive
+	archiveName := archiveName(archiveNameHint, archiveInput)
+
+	// Get files from inputs as number of files decide next steps
+	files, err := utils.FindAllGlobs(archiveInput)
+	if err != nil {
+		util.CheckErr(err, "finding all globs")
+	}
+
+	// We have one file; if it's a zip file or a URL, no need to archive it
+	if len(files) == 1 {
+		// make sure it exists
+		if _, err := os.Stat(files[0]); err != nil {
+			util.CheckErr(err, fmt.Sprintf("open input file %v", files[0]))
+		}
+
+		// if it's an existing zip file OR we're not supposed to zip it, don't do anything
+		if archiver.Zip.Match(files[0]) || noZip {
+			return files[0]
+		}
+
+		// if it's an HTTP URL, just use the URL.
+		if strings.HasPrefix(files[0], "http://") || strings.HasPrefix(files[0], "https://") {
+			return files[0]
+		}
+	}
+
+	// For anything else, create a new archive
+	tmpDir, err := utils.GetTempDir()
+	if err != nil {
+		util.CheckErr(err, "create temporary archive directory")
+	}
+
+	archivePath, err := utils.MakeArchive(filepath.Join(tmpDir, archiveName), archiveInput...)
+	if err != nil {
+		util.CheckErr(err, "create archive file")
+	}
+
+	return archivePath
+}
+
+// Name an archive
+func archiveName(givenNameHint string, includedFiles []string) string {
+	if len(givenNameHint) > 0 {
+		return fmt.Sprintf("%v-%v", givenNameHint, uniuri.NewLen(4))
+	}
+	if len(includedFiles) == 0 {
+		return uniuri.NewLen(8)
+	}
+	return fmt.Sprintf("%v-%v", util.KubifyName(includedFiles[0]), uniuri.NewLen(4))
+}

pkg/fission-cli/plugin.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+
+	"github.com/urfave/cli"
+
+	"github.com/fission/fission/pkg/fission-cli/plugin"
+)
+
+var cmdPlugin = cli.Command{
+	Name:    "plugin",
+	Aliases: []string{"plugins"},
+	Usage:   "Manage Fission CLI plugins",
+	Subcommands: []cli.Command{
+		{
+			Name:   "list",
+			Usage:  "List installed client plugins",
+			Action: pluginList,
+		},
+	},
+}
+
+func pluginList(_ *cli.Context) error {
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintln(w, "NAME\tVERSION\tPATH")
+	for _, p := range plugin.FindAll() {
+		fmt.Fprintf(w, "%v\t%v\t%v\n", p.Name, p.Version, p.Path)
+	}
+	w.Flush()
+	return nil
+}

pkg/fission-cli/plugin/plugin.go

@@ -0,0 +1,179 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package plugins provides support for creating extensible CLIs
+package plugin
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"time"
+)
+
+const (
+	cmdTimeout      = 5 * time.Second
+	cmdMetadataArgs = "--plugin"
+)
+
+var (
+	ErrPluginNotFound = errors.New("plugin not found")
+	ErrPluginInvalid  = errors.New("invalid plugin")
+
+	Prefix = "fission-"
+)
+
+// Metadata contains the metadata of a plugin.
+// The only metadata that is guaranteed to be non-empty is the path and Name. All other fields are considered optional.
+type Metadata struct {
+	Name    string   `json:"name,omitempty"`
+	Version string   `json:"version,omitempty"`
+	Aliases []string `json:"aliases,omitempty"`
+	Usage   string   `json:"usage,omitempty"`
+	Path    string   `json:"path,omitempty"`
+}
+
+func (md *Metadata) AddAlias(alias string) {
+	if alias != md.Name && !md.HasAlias(alias) {
+		md.Aliases = append(md.Aliases, alias)
+	}
+}
+
+func (md *Metadata) HasAlias(needle string) bool {
+	for _, alias := range md.Aliases {
+		if alias == needle {
+			return true
+		}
+	}
+	return false
+}
+
+// Find searches the machine for the given plugin, returning the metadata of the plugin.
+// The only metadata that is guaranteed to be non-empty is the path and Name. All other fields are considered optional.
+// If found it returns the plugin, otherwise it returns ErrPluginNotFound if the plugin was not found.
+func Find(pluginName string) (*Metadata, error) {
+	// Search PATH for plugin as command-name
+	// To check if plugin is actually there still.
+	pluginPath, err := findPluginOnPath(pluginName)
+	if err != nil {
+		// Fallback: Search for alias in each command
+		mds := FindAll()
+		for _, md := range mds {
+			if md.HasAlias(pluginName) {
+				return md, nil
+			}
+		}
+		return nil, ErrPluginNotFound
+	}
+
+	md, err := fetchPluginMetadata(pluginPath)
+	if err != nil {
+		return nil, err
+	}
+	return md, nil
+}
+
+// Exec executes the plugin using the provided args.
+// All input and output is redirected to stdin, stdout, and stderr.
+func Exec(md *Metadata, args []string) error {
+	cmd := exec.Command(md.Path, args...)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+// FindAll searches the machine for all plugins currently present.
+func FindAll() map[string]*Metadata {
+	plugins := map[string]*Metadata{}
+
+	dirs := strings.Split(os.Getenv("PATH"), ":")
+	for _, dir := range dirs {
+		fs, err := ioutil.ReadDir(dir)
+		if err != nil {
+			continue
+		}
+		for _, f := range fs {
+			if !strings.HasPrefix(f.Name(), Prefix) {
+				continue
+			}
+			fp := path.Join(dir, f.Name())
+			md, err := fetchPluginMetadata(fp)
+			if err != nil {
+				continue
+			}
+			if existing, ok := plugins[md.Name]; ok {
+				for _, alias := range existing.Aliases {
+					md.AddAlias(alias)
+				}
+			}
+			plugins[md.Name] = md
+		}
+	}
+	return plugins
+}
+
+func findPluginOnPath(pluginName string) (path string, err error) {
+	binaryName := Prefix + pluginName
+	path, err = exec.LookPath(binaryName)
+
+	if err != nil || len(path) == 0 {
+		return "", ErrPluginNotFound
+	}
+	return path, nil
+}
+
+// fetchPluginMetadata attempts to fetch the plugin metadata given the plugin path.
+func fetchPluginMetadata(pluginPath string) (*Metadata, error) {
+	d, err := os.Stat(pluginPath)
+	if err != nil {
+		return nil, ErrPluginNotFound
+	}
+	if m := d.Mode(); m.IsDir() || m&0111 == 0 {
+		return nil, ErrPluginInvalid
+	}
+
+	// Fetch the metadata from the plugin itself.
+	buf := bytes.NewBuffer(nil)
+	ctx, cancel := context.WithTimeout(context.Background(), cmdTimeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, pluginPath, cmdMetadataArgs) // Note: issue can occur with signal propagation
+	cmd.Stdout = buf
+	err = cmd.Run()
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse metadata if possible
+	pluginName := strings.TrimPrefix(path.Base(pluginPath), Prefix)
+	md := &Metadata{}
+	err = json.Unmarshal(buf.Bytes(), md)
+
+	// If metadata could not be retrieved, or if no name was provided, use the filename of the binary
+	if err != nil || len(md.Name) == 0 {
+		md.Name = pluginName
+	}
+	md.Path = pluginPath
+	md.AddAlias(pluginName)
+	return md, nil
+}

pkg/fission-cli/plugin/plugin_test.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package plugin
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFind(t *testing.T) {
+	os.Clearenv()
+	testDir := path.Join(os.TempDir(), fmt.Sprintf("fission-test-plugins-%v", time.Now().UnixNano()))
+	err := os.MkdirAll(testDir, os.ModePerm)
+	if err != nil {
+		t.FailNow()
+	}
+	defer os.RemoveAll(testDir)
+	testBinary := path.Join(testDir, "foo")
+	md := &Metadata{
+		Name:    "foo",
+		Version: "1.0.1",
+		Usage:   "Usage help",
+		Aliases: []string{"bar"},
+	}
+	jsonMd, err := json.Marshal(md)
+	if err != nil {
+		t.FailNow()
+	}
+	err = ioutil.WriteFile(testBinary, []byte(fmt.Sprintf("#!/bin/sh\necho '%v'", string(jsonMd))), os.ModePerm)
+	if err != nil {
+		t.FailNow()
+	}
+
+	err = os.Setenv("PATH", testDir)
+	if err != nil {
+		t.FailNow()
+	}
+	Prefix = ""
+
+	found, err := Find(md.Name)
+	os.RemoveAll(testDir)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, found)
+	assert.Equal(t, md.Name, found.Name)
+	assert.Equal(t, path.Join(testDir, md.Name), found.Path)
+	assert.Equal(t, md.Aliases, found.Aliases)
+	assert.Equal(t, md.Usage, found.Usage)
+	assert.Equal(t, md.Version, found.Version)
+}
+
+func TestExec(t *testing.T) {
+	os.Clearenv()
+	testDir := path.Join(os.TempDir(), fmt.Sprintf("fission-test-plugins-%v", time.Now().UnixNano()))
+	err := os.MkdirAll(testDir, os.ModePerm)
+	if err != nil {
+		t.FailNow()
+	}
+	defer os.RemoveAll(testDir)
+	testBinary := path.Join(testDir, "foo")
+	md := &Metadata{
+		Name:    "foo",
+		Version: "1.0.1",
+		Usage:   "Usage help",
+		Aliases: []string{"bar"},
+		Path:    path.Join(testBinary),
+	}
+	jsonMd, err := json.Marshal(md)
+	if err != nil {
+		t.FailNow()
+	}
+	err = ioutil.WriteFile(testBinary, []byte(fmt.Sprintf("#!/bin/sh\necho '%v'", string(jsonMd))), os.ModePerm)
+	if err != nil {
+		t.FailNow()
+	}
+	err = os.Setenv("PATH", testDir)
+	if err != nil {
+		t.FailNow()
+	}
+	Prefix = ""
+	err = Exec(md, nil)
+	os.RemoveAll(testDir)
+	assert.NoError(t, err)
+}

pkg/fission-cli/plugin/registry.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package plugin
+
+// builtinRegistry consists of a map of plugin names along with the relevant url.
+var builtinRegistry = map[string]string{
+	"workflows": "https://github.com/fission/fission-workflows/releases",
+}
+
+// SearchRegistries will search (remote) registries for the presence of the command.
+// For now we only use the builtinRegistry
+func SearchRegistries(cmd string) (string, bool) {
+	url, ok := builtinRegistry[cmd]
+	return url, ok
+}

pkg/fission-cli/recorder.go

@@ -0,0 +1,238 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    tttp://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/satori/go.uuid"
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/util"
+)
+
+func recorderCreate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	recName := c.String("name")
+	if len(recName) == 0 {
+		recName = uuid.NewV4().String()
+	}
+	fnName := c.String("function")
+	triggersOriginal := c.StringSlice("trigger")
+
+	// Function XOR triggers can be given
+	if len(fnName) == 0 && len(triggersOriginal) == 0 {
+		log.Fatal("Need to specify at least one function or one trigger, use --function, --trigger")
+	}
+	if len(fnName) != 0 && len(triggersOriginal) != 0 {
+		log.Fatal("Can specify either one function or one or more triggers, but not both")
+	}
+
+	// TODO: Validate here or elsewhere that all triggers belong to the same namespace
+
+	var triggers []string
+	if len(triggersOriginal) != 0 {
+		ts := strings.Split(triggersOriginal[0], ",")
+		for _, name := range ts {
+			triggers = append(triggers, name)
+		}
+	}
+	// TODO: Define appropriate set of policies and defaults
+	//retPolicy := c.String("retention")
+	//evictPolicy := c.String("eviction")
+
+	recorder := &fv1.Recorder{
+		Metadata: metav1.ObjectMeta{
+			Name:      recName,
+			Namespace: "default",
+		},
+		Spec: fv1.RecorderSpec{
+			Name:            recName,
+			Function:        fnName,
+			Triggers:        triggers,
+			RetentionPolicy: "Permanent", // TODO: Implement customizable policies for expiration of records
+			EvictionPolicy:  "None",
+			Enabled:         true,
+		},
+	}
+
+	// If we're writing a spec, don't call the API
+	if c.Bool("spec") {
+		specFile := fmt.Sprintf("recorder-%v.yaml", recName)
+		err := specSave(*recorder, specFile)
+		util.CheckErr(err, "create recorder spec")
+		return nil
+	}
+
+	_, err := client.RecorderCreate(recorder)
+	util.CheckErr(err, "create recorder")
+
+	fmt.Printf("recorder '%s' created\n", recName)
+	return err
+}
+
+func recorderGet(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	recName := c.String("name")
+
+	recorder, err := client.RecorderGet(&metav1.ObjectMeta{
+		Name:      recName,
+		Namespace: "default",
+	})
+
+	util.CheckErr(err, "get recorder")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n",
+		"NAME", "ENABLED", "FUNCTION", "TRIGGERS", "RETENTION_POLICY", "EVICTION_POLICY")
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n",
+		recorder.Metadata.Name, recorder.Spec.Enabled, recorder.Spec.Function, recorder.Spec.Triggers, recorder.Spec.RetentionPolicy, recorder.Spec.EvictionPolicy)
+	w.Flush()
+
+	return nil
+}
+
+func recorderUpdate(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	recName := c.String("name")
+	enable := c.Bool("enable")
+	disable := c.Bool("disable")
+	//retPolicy := c.String("retention")
+	//evictPolicy := c.String("eviction")
+	triggers := c.StringSlice("trigger")
+	function := c.String("function")
+
+	if enable && disable {
+		log.Fatal("Cannot enable and disable a recorder simultaneously.")
+	}
+
+	// Prevent enable or disable while trying to update other fields. These flags must be standalone.
+	if enable || disable {
+		if len(triggers) > 0 || len(function) > 0 {
+			log.Fatal("Enabling or disabling a recorder with other (non-name) flags set is not supported.")
+		}
+	} else if len(triggers) == 0 && len(function) == 0 {
+		log.Fatal("Need to specify either a function or trigger(s) for this recorder")
+	}
+
+	if len(recName) == 0 {
+		log.Fatal("Need name of recorder, use --name")
+	}
+
+	recorder, err := client.RecorderGet(&metav1.ObjectMeta{
+		Name:      recName,
+		Namespace: "default",
+	})
+
+	updated := false
+
+	// TODO: Additional validation on type of supported retention policy, eviction policy
+
+	//if len(retPolicy) > 0 {
+	//	recorder.Spec.RetentionPolicy = retPolicy
+	//	updated = true
+	//}
+	//if len(evictPolicy) > 0 {
+	//	recorder.Spec.EvictionPolicy = evictPolicy
+	//	updated = true
+	//}
+	if enable {
+		recorder.Spec.Enabled = true
+		updated = true
+	}
+
+	if disable {
+		recorder.Spec.Enabled = false
+		updated = true
+	}
+
+	if len(triggers) > 0 {
+		var newTriggers []string
+		triggs := strings.Split(triggers[0], ",")
+		for _, name := range triggs {
+			newTriggers = append(newTriggers, name)
+		}
+		recorder.Spec.Triggers = newTriggers
+		updated = true
+	}
+
+	if len(function) > 0 {
+		recorder.Spec.Function = function
+		updated = true
+	}
+
+	if !updated {
+		log.Fatal("Nothing to update. Use --function, --triggers, --enable or --disable")
+	}
+
+	_, err = client.RecorderUpdate(recorder)
+	util.CheckErr(err, "update recorder")
+
+	fmt.Printf("recorder '%v' updated\n", recName)
+	return nil
+}
+
+func recorderDelete(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	recName := c.String("name")
+
+	if len(recName) == 0 {
+		log.Fatal("Need name of recorder to delete, use --name")
+	}
+
+	recNs := c.String("recorderns")
+
+	err := client.RecorderDelete(&metav1.ObjectMeta{
+		Name:      recName,
+		Namespace: recNs,
+	})
+
+	util.CheckErr(err, "delete recorder")
+
+	fmt.Printf("recorder '%v' deleted\n", recName)
+	return nil
+}
+
+func recorderList(c *cli.Context) error {
+	client := util.GetApiClient(c.GlobalString("server"))
+
+	recorders, err := client.RecorderList("default")
+	util.CheckErr(err, "list recorders")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n",
+		"NAME", "ENABLED", "FUNCTIONS", "TRIGGERS", "RETENTION_POLICY", "EVICTION_POLICY")
+	for _, r := range recorders {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\n",
+			r.Metadata.Name, r.Spec.Enabled, r.Spec.Function, r.Spec.Triggers, r.Spec.RetentionPolicy, r.Spec.EvictionPolicy)
+	}
+	w.Flush()
+
+	return nil
+}

pkg/fission-cli/records.go

@@ -0,0 +1,145 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    tttp://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+
+	"github.com/urfave/cli"
+
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/util"
+	"github.com/fission/fission/pkg/redis/build/gen"
+)
+
+func recordsView(c *cli.Context) error {
+	var verbosity int
+	if c.Bool("v") && c.Bool("vv") {
+		log.Fatal("conflicting verbosity levels, use either --v or --vv")
+	}
+	if c.Bool("v") {
+		verbosity = 1
+	}
+	if c.Bool("vv") {
+		verbosity = 2
+	}
+
+	function := c.String("function")
+	trigger := c.String("trigger")
+	from := c.String("from")
+	to := c.String("to")
+
+	//Refuse multiple filters for now
+	if multipleFiltersSpecified(function, trigger, from+to) {
+		log.Fatal("maximum of one filter is currently supported, either --function, --trigger, or --from,--to")
+	}
+
+	if len(function) != 0 {
+		return recordsByFunction(function, verbosity, c)
+	}
+	if len(trigger) != 0 {
+		return recordsByTrigger(trigger, verbosity, c)
+	}
+	if len(from) != 0 && len(to) != 0 {
+		return recordsByTime(from, to, verbosity, c)
+	}
+	err := recordsAll(verbosity, c)
+	util.CheckErr(err, "view records")
+	return nil
+}
+
+func recordsAll(verbosity int, c *cli.Context) error {
+	fc := util.GetApiClient(c.GlobalString("server"))
+
+	records, err := fc.RecordsAll()
+	util.CheckErr(err, "view records")
+
+	showRecords(records, verbosity)
+
+	return nil
+}
+
+func recordsByTrigger(trigger string, verbosity int, c *cli.Context) error {
+	fc := util.GetApiClient(c.GlobalString("server"))
+
+	records, err := fc.RecordsByTrigger(trigger)
+	util.CheckErr(err, "view records")
+
+	showRecords(records, verbosity)
+
+	return nil
+}
+
+// TODO: More accurate function name (function filter)
+func recordsByFunction(function string, verbosity int, c *cli.Context) error {
+	fc := util.GetApiClient(c.GlobalString("server"))
+
+	records, err := fc.RecordsByFunction(function)
+	util.CheckErr(err, "view records")
+
+	showRecords(records, verbosity)
+
+	return nil
+}
+
+func recordsByTime(from string, to string, verbosity int, c *cli.Context) error {
+	fc := util.GetApiClient(c.GlobalString("server"))
+
+	records, err := fc.RecordsByTime(from, to)
+	util.CheckErr(err, "view records")
+
+	showRecords(records, verbosity)
+
+	return nil
+}
+
+func showRecords(records []*redisCache.RecordedEntry, verbosity int) {
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+
+	if verbosity == 1 {
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n",
+			"REQUID", "REQUEST METHOD", "FUNCTION", "RESPONSE STATUS", "TRIGGER")
+		for _, record := range records {
+			fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n",
+				record.ReqUID, record.Req.Method, record.Req.Header["X-Fission-Function-Name"], record.Resp.Status, record.Trigger)
+		}
+	} else if verbosity == 2 {
+		for _, record := range records {
+			fmt.Println(record)
+		}
+	} else {
+		fmt.Fprintf(w, "%v\n",
+			"REQUID")
+		for _, record := range records {
+			fmt.Fprintf(w, "%v\n",
+				record.ReqUID)
+		}
+	}
+	w.Flush()
+}
+
+func multipleFiltersSpecified(entries ...string) bool {
+	var specified int
+	for _, entry := range entries {
+		if len(entry) > 0 {
+			specified += 1
+		}
+	}
+	return specified > 1
+}

pkg/fission-cli/replay.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fission_cli
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/fission-cli/util"
+	"github.com/urfave/cli"
+)
+
+func replay(c *cli.Context) error {
+	fc := util.GetApiClient(c.GlobalString("server"))
+
+	reqUID := c.String("reqUID")
+	if len(reqUID) == 0 {
+		log.Fatal("Need a reqUID, use --reqUID flag to specify")
+	}
+
+	responses, err := fc.ReplayByReqUID(reqUID)
+	util.CheckErr(err, "replay records")
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+
+	for _, resp := range responses {
+		fmt.Fprintf(w, "%v",
+			resp,
+		)
+	}
+
+	w.Flush()
+
+	return nil
+}

pkg/fission-cli/support/dump.go

@@ -0,0 +1,141 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package support
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+
+	"github.com/fission/fission/pkg/fission-cli/support/resources"
+	"github.com/fission/fission/pkg/fission-cli/util"
+	"github.com/fission/fission/pkg/utils"
+)
+
+const (
+	DUMP_ARCHIVE_PREFIX = "fission-dump"
+	DEFAULT_OUTPUT_DIR  = "fission-dump"
+)
+
+func DumpInfo(c *cli.Context) error {
+
+	fmt.Println("Start dumping process...")
+
+	nozip := c.Bool("nozip")
+	outputDir := c.String("output")
+
+	// check whether the dump directory exists.
+	_, err := os.Stat(outputDir)
+	if err != nil && os.IsNotExist(err) {
+		err = os.Mkdir(outputDir, 0755)
+		if err != nil {
+			panic(err)
+		}
+	} else if err != nil {
+		panic(errors.Wrap(err, "Error checking dump directory status"))
+	}
+
+	outputDir, err = filepath.Abs(outputDir)
+	if err != nil {
+		panic(errors.Wrap(err, "Error creating dump directory for dumping files"))
+	}
+
+	client := util.GetApiClient(util.GetServerUrl())
+	_, k8sClient := util.GetKubernetesClient()
+
+	ress := map[string]resources.Resource{
+		// kubernetes info
+		"kubernetes-version": resources.NewKubernetesVersion(k8sClient),
+		"kubernetes-nodes":   resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesNode, ""),
+
+		// fission info
+		"fission-version": resources.NewFissionVersion(client),
+
+		// fission component logs & spec
+		"fission-components-svc-sepc": resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesService,
+			"svc in (buildermgr, controller, executor, influxdb, kubewatcher, logger, mqtrigger, nats-streaming, redis, router, storagesvc, timer)"),
+		"fission-components-deployment-sepc": resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesDeployment,
+			"svc in (buildermgr, controller, executor, influxdb, kubewatcher, logger, mqtrigger, nats-streaming, redis, router, storagesvc, timer)"),
+		"fission-components-pod-sepc": resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesPod,
+			"svc in (buildermgr, controller, executor, influxdb, kubewatcher, logger, mqtrigger, nats-streaming, redis, router, storagesvc, timer)"),
+		"fission-components-pod-log": resources.NewKubernetesPodLogDumper(k8sClient,
+			"svc in (buildermgr, controller, executor, influxdb, kubewatcher, logger, mqtrigger, nats-streaming, redis, router, storagesvc, timer)"),
+
+		// fission builder logs & spec
+		"fission-builder-svc-spec":        resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesService, "owner=buildermgr"),
+		"fission-builder-deployment-spec": resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesDeployment, "owner=buildermgr"),
+		"fission-builder-pod-spec":        resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesPod, "owner=buildermgr"),
+		"fission-builder-pod-log":         resources.NewKubernetesPodLogDumper(k8sClient, "owner=buildermgr"),
+
+		// fission function logs & spec
+		"fission-function-svc-sepc":        resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesService, "executorType=newdeploy"),
+		"fission-function-deployment-spec": resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesDeployment, "executorType in (poolmgr, newdeploy)"),
+		"fission-function-pod-spec":        resources.NewKubernetesObjectDumper(k8sClient, resources.KubernetesPod, "executorType in (poolmgr, newdeploy)"),
+		"fission-function-pod-log":         resources.NewKubernetesPodLogDumper(k8sClient, "executorType in (poolmgr, newdeploy)"),
+
+		// CRD resources
+		"fission-crd-packages":     resources.NewCrdDumper(client, resources.CrdPackage),
+		"fission-crd-environments": resources.NewCrdDumper(client, resources.CrdEnvironment),
+		"fission-crd-functions":    resources.NewCrdDumper(client, resources.CrdFunction),
+		"fission-crd-httptriggers": resources.NewCrdDumper(client, resources.CrdHttpTrigger),
+		"fission-crd-kubewatchers": resources.NewCrdDumper(client, resources.CrdKubeWatcher),
+		"fission-crd-mqtriggers":   resources.NewCrdDumper(client, resources.CrdMessageQueueTrigger),
+		"fission-crd-timetriggers": resources.NewCrdDumper(client, resources.CrdTimeTrigger),
+	}
+
+	dumpName := fmt.Sprintf("%v_%v", DUMP_ARCHIVE_PREFIX, time.Now().Unix())
+	dumpDir := filepath.Join(outputDir, dumpName)
+
+	wg := &sync.WaitGroup{}
+
+	for key, res := range ress {
+		dir := fmt.Sprintf("%v/%v/", dumpDir, key)
+		if _, err := os.Stat(dir); os.IsNotExist(err) {
+			err = os.MkdirAll(dir, 0755)
+			if err != nil {
+				panic(err)
+			}
+		}
+		go func(res resources.Resource, dir string) {
+			wg.Add(1)
+			defer wg.Done()
+			res.Dump(dir)
+		}(res, dir)
+	}
+
+	wg.Wait()
+
+	if !nozip {
+		defer os.Remove(dumpDir)
+		path := filepath.Join(outputDir, fmt.Sprintf("%v.zip", dumpName))
+		_, err := utils.MakeArchive(path, dumpDir)
+		if err != nil {
+			fmt.Printf("Error creating archive for dump files: %v", err)
+			return nil
+		}
+		fmt.Printf("The archive dump file is %v\n", path)
+	} else {
+		fmt.Printf("The dump files are placed at %v\n", dumpDir)
+	}
+
+	return nil
+}

pkg/fission-cli/support/resources/crd.go

@@ -0,0 +1,158 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resources
+
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/controller/client"
+	"github.com/fission/fission/pkg/fission-cli/log"
+	"github.com/fission/fission/pkg/types"
+)
+
+const (
+	CrdEnvironment = "Environment"
+	CrdFunction    = "Function"
+	CrdPackage     = "Packages"
+
+	CrdHttpTrigger         = "HTTPTrigger"
+	CrdKubeWatcher         = "KubeWatcher"
+	CrdMessageQueueTrigger = "MessageQueue"
+	CrdTimeTrigger         = "TimeTrigger"
+)
+
+type CrdDumper struct {
+	client  *client.Client
+	crdType string
+}
+
+func NewCrdDumper(client *client.Client, crdType string) Resource {
+	return CrdDumper{client: client, crdType: crdType}
+}
+
+func (res CrdDumper) Dump(dumpDir string) {
+
+	switch res.crdType {
+	case CrdEnvironment:
+		items, err := res.client.EnvironmentList(metav1.NamespaceAll)
+		if err != nil {
+			log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
+			return
+		}
+
+		for _, item := range items {
+			f := getFileName(dumpDir, item.Metadata)
+			writeToFile(f, item)
+		}
+
+	case CrdFunction:
+		items, err := res.client.FunctionList(metav1.NamespaceAll)
+		if err != nil {
+			log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
+			return
+		}
+
+		for _, item := range items {
+			f := getFileName(dumpDir, item.Metadata)
+			writeToFile(f, item)
+		}
+
+	case CrdPackage:
+		items, err := res.client.PackageList(metav1.NamespaceAll)
+		if err != nil {
+			log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
+			return
+		}
+
+		for _, item := range items {
+			item = pkgClean(item)
+			f := getFileName(dumpDir, item.Metadata)
+			writeToFile(f, item)
+		}
+
+	case CrdHttpTrigger:
+		items, err := res.client.HTTPTriggerList(metav1.NamespaceAll)
+		if err != nil {
+			log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
+			return
+		}
+
+		for _, item := range items {
+			f := getFileName(dumpDir, item.Metadata)
+			writeToFile(f, item)
+		}
+
+	case CrdKubeWatcher:
+		items, err := res.client.WatchList(metav1.NamespaceAll)
+		if err != nil {
+			log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
+			return
+		}
+
+		for _, item := range items {
+			f := getFileName(dumpDir, item.Metadata)
+			writeToFile(f, item)
+		}
+
+	case CrdMessageQueueTrigger:
+		var triggers []fv1.MessageQueueTrigger
+
+		for _, mqType := range []string{types.MessageQueueTypeNats, types.MessageQueueTypeASQ} {
+			l, err := res.client.MessageQueueTriggerList(mqType, metav1.NamespaceAll)
+			if err != nil {
+				log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
+				break
+			}
+			triggers = append(triggers, l...)
+		}
+
+		for _, item := range triggers {
+			f := getFileName(dumpDir, item.Metadata)
+			writeToFile(f, item)
+		}
+
+	case CrdTimeTrigger:
+		items, err := res.client.TimeTriggerList(metav1.NamespaceAll)
+		if err != nil {
+			log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
+			return
+		}
+
+		for _, item := range items {
+			f := getFileName(dumpDir, item.Metadata)
+			writeToFile(f, item)
+		}
+
+	default:
+		log.Info(fmt.Sprintf("Unknown type: %v", res.crdType))
+	}
+}
+
+func pkgClean(pkg fv1.Package) fv1.Package {
+	// mask the sensitive information
+	// use "-" as mask value to indicate the field wasn't empty
+	if pkg.Spec.Source.Literal != nil {
+		pkg.Spec.Source.Literal = []byte("-")
+	}
+	if pkg.Spec.Deployment.Literal != nil {
+		pkg.Spec.Deployment.Literal = []byte("-")
+	}
+	return pkg
+}