Version -> 0.4.1

Remove hard coded field indices in influx query, instead searching for them by name.

.travis.yml

@@ -12,6 +12,13 @@ cache:
     - $HOME/google-cloud-sdk/
     - $HOME/k8scli
 
+services:
+  - docker
+
+before_install:
+  - sudo apt-get update
+  - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
+
 install:
   - go get github.com/Masterminds/glide
   - hack/travis-kube-setup.sh

Documentation/docs-site/content/install.md

@@ -75,7 +75,7 @@ $ helm init
 #### Minikube
 
 ```
-$ helm install --namespace fission --set serviceType=NodePort https://github.com/fission/fission/releases/download/0.4.0/fission-all-0.4.0.tgz
+$ helm install --namespace fission --set serviceType=NodePort https://github.com/fission/fission/releases/download/0.4.1/fission-all-0.4.1.tgz
 ```
 
 The serviceType variable allows configuring the type of Kubernetes
@@ -85,7 +85,7 @@ want to expose anything outside the cluster.
 #### Cloud hosted clusters (GKE, AWS, Azure etc.)
 
 ```
-$ helm install --namespace fission https://github.com/fission/fission/releases/download/0.4.0/fission-all-0.4.0.tgz
+$ helm install --namespace fission https://github.com/fission/fission/releases/download/0.4.1/fission-all-0.4.1.tgz
 ```
 
 #### Minimal version
@@ -95,7 +95,7 @@ the NATS message queue, influxDB for logs, etc. If you want a more
 minimal setup, you can install the fission-core chart instead:
 
 ```
-$ helm install --namespace fission https://github.com/fission/fission/releases/download/0.4.0/fission-core-0.4.0.tgz
+$ helm install --namespace fission https://github.com/fission/fission/releases/download/0.4.1/fission-core-0.4.1.tgz
 ```
 
 ### Install the Fission CLI
@@ -105,19 +105,19 @@ $ helm install --namespace fission https://github.com/fission/fission/releases/d
 Get the CLI binary for Mac:
 
 ```
-$ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.0/fission-cli-osx && chmod +x fission && sudo mv fission /usr/local/bin/
+$ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.1/fission-cli-osx && chmod +x fission && sudo mv fission /usr/local/bin/
 ```
 
 #### Linux
 
 ```
-$ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.0/fission-cli-linux && chmod +x fission && sudo mv fission /usr/local/bin/
+$ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.1/fission-cli-linux && chmod +x fission && sudo mv fission /usr/local/bin/
 ```
 
 #### Windows
 
 For Windows, you can use the linux binary on WSL. Or you can download
-this windows executable: [fission.exe](https://github.com/fission/fission/releases/download/0.4.0/fission-cli-windows.exe)
+this windows executable: [fission.exe](https://github.com/fission/fission/releases/download/0.4.1/fission-cli-windows.exe)
 
 ### Set environment vars
 
@@ -157,7 +157,7 @@ svc```).  Then:
 Finally, you're ready to use Fission!
 
 ```
-$ fission env create --name nodejs --image fission/node-env:0.4.0
+$ fission env create --name nodejs --image fission/node-env:0.4.1
 
 $ curl -LO https://raw.githubusercontent.com/fission/fission/master/examples/nodejs/hello.js
 

README.md

@@ -97,7 +97,7 @@ See the [examples](examples) directory for more.
 Running Fission on your Cluster
 ===============================
 
-See the [installation guide](http://fission.io/docs/v0.2.1/install/).
+See the [installation guide](http://fission.io/docs/0.4.0/install/).
 
 Compiling Fission
 =================

builder/builder.go

@@ -21,11 +21,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"strings"
 	"time"
@@ -68,7 +70,9 @@ func MakeBuilder(sharedVolumePath string) *Builder {
 
 func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) {
 	if r.Method != "POST" {
-		http.Error(w, "", 405)
+		e := fmt.Sprintf("Method not allowed: %v", r.Method)
+		log.Println(e)
+		builder.reply(w, "", e, http.StatusMethodNotAllowed)
 		return
 	}
 
@@ -81,15 +85,17 @@ func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) {
 	// parse request
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
-		log.Printf("Error reading request body")
+		e := errors.New(fmt.Sprintf("Error reading request body: %v", err))
-		http.Error(w, err.Error(), 500)
+		log.Println(e.Error())
+		builder.reply(w, "", e.Error(), http.StatusInternalServerError)
 		return
 	}
 	var req PackageBuildRequest
 	err = json.Unmarshal(body, &req)
 	if err != nil {
-		log.Printf("Error parsing json body: %v", err)
+		e := errors.New(fmt.Sprintf("Error parsing json body: %v", err))
-		http.Error(w, err.Error(), 400)
+		log.Println(e.Error())
+		builder.reply(w, "", e.Error(), http.StatusBadRequest)
 		return
 	}
 	log.Printf("Builder received request: %v", req)
@@ -106,66 +112,100 @@ func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) {
 	buildLogs, err := builder.build(buildCmd, srcPkgPath, deployPkgPath)
 	if err != nil {
 		e := errors.New(fmt.Sprintf("Error building source package: %v", err))
-		http.Error(w, e.Error(), 500)
+		log.Println(e.Error())
+		// append error at the end of build logs
+		buildLogs += fmt.Sprintf("%v\n", e.Error())
+		builder.reply(w, deployPkgFilename, buildLogs, http.StatusInternalServerError)
 		return
 	}
 
+	builder.reply(w, deployPkgFilename, buildLogs, http.StatusOK)
+}
+
+func (builder *Builder) reply(w http.ResponseWriter, pkgFilename string, buildLogs string, statusCode int) {
 	resp := PackageBuildResponse{
-		ArtifactFilename: deployPkgFilename,
+		ArtifactFilename: pkgFilename,
 		BuildLogs:        buildLogs,
 	}
 
 	rBody, err := json.Marshal(resp)
 	if err != nil {
 		e := errors.New(fmt.Sprintf("Error encoding response body: %v", err))
-		http.Error(w, e.Error(), 500)
+		rBody = []byte(fmt.Sprintf(`{"buildLogs": "%v"}`, e.Error()))
-		return
+		statusCode = http.StatusInternalServerError
 	}
 
 	w.Header().Add("Content-Type", "application/json")
+	// should write header before writing the body,
+	// or client will receive HTTP 200 regardless the real status code
+	w.WriteHeader(statusCode)
 	w.Write(rBody)
-	w.WriteHeader(http.StatusOK)
 }
 
 func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath string) (string, error) {
 	cmd := exec.Command(command)
-	cmd.Dir = srcPkgPath
+
+	fi, err := os.Stat(srcPkgPath)
+	if err != nil {
+		return "", errors.New(fmt.Sprintf("could not find srcPkgPath: '%s'", srcPkgPath))
+	}
+	if fi.IsDir() {
+		cmd.Dir = srcPkgPath
+	} else {
+		cmd.Dir = path.Dir(srcPkgPath)
+	}
+
 	// set env variables for build command
 	cmd.Env = append(os.Environ(),
 		fmt.Sprintf("%v=%v", envSrcPkg, srcPkgPath),
 		fmt.Sprintf("%v=%v", envDeployPkg, deployPkgPath),
 	)
 
-	cmdReader, err := cmd.StdoutPipe()
+	stdout, err := cmd.StdoutPipe()
 	if err != nil {
 		return "", errors.New(fmt.Sprintf("Error creating stdout pipe for cmd: %v", err.Error()))
 	}
 
-	scanner := bufio.NewScanner(cmdReader)
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return "", errors.New(fmt.Sprintf("Error creating stderr pipe for cmd: %v", err.Error()))
+	}
+
+	var buildLogs string
+
+	fmt.Println("\n=== Build Logs ===")
+	// Init logs
+	fmt.Printf("command=%v\n", command)
+	fmt.Printf("env=%v\n", cmd.Env)
+
+	out := io.MultiReader(stdout, stderr)
+	scanner := bufio.NewScanner(out)
 
 	err = cmd.Start()
 	if err != nil {
 		return "", errors.New(fmt.Sprintf("Error starting cmd: %v", err.Error()))
 	}
 
-	var buildLogs string
+	// Runtime logs
-
-	fmt.Println("\n=== Build Logs ===")
 	for scanner.Scan() {
 		output := scanner.Text()
 		fmt.Println(output)
 		buildLogs += fmt.Sprintf("%v\n", output)
 	}
-	fmt.Println("==================\n")
 
 	if err := scanner.Err(); err != nil {
-		return "", errors.New(fmt.Sprintf("Error reading cmd output: %v", err.Error()))
+		scanErr := errors.New(fmt.Sprintf("Error reading cmd output: %v", err.Error()))
+		fmt.Println(scanErr)
+		return buildLogs, scanErr
 	}
 
 	err = cmd.Wait()
 	if err != nil {
-		return "", errors.New(fmt.Sprintf("Error waiting for cmd: %v", err.Error()))
+		cmdErr := errors.New(fmt.Sprintf("Error waiting for cmd '%v': %v", command, err.Error()))
+		fmt.Println(cmdErr)
+		return buildLogs, cmdErr
 	}
+	fmt.Println("==================\n")
 
 	return buildLogs, nil
 }

builder/client/client.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"strings"
 
@@ -50,20 +51,18 @@ func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildR
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode != 200 {
-		return nil, fission.MakeErrorFromHTTP(resp)
-	}
-
 	rBody, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
+		log.Printf("Error reading resp body: %v", err)
 		return nil, err
 	}
 
 	pkgBuildResp := builder.PackageBuildResponse{}
 	err = json.Unmarshal([]byte(rBody), &pkgBuildResp)
 	if err != nil {
+		log.Printf("Error parsing resp body: %v", err)
 		return nil, err
 	}
 
-	return &pkgBuildResp, nil
+	return &pkgBuildResp, fission.MakeErrorFromHTTP(resp)
 }

builder/cmd/Dockerfile

@@ -0,0 +1,9 @@
+FROM alpine:3.5
+
+RUN apk update
+# Install some utility libraries for builders extending image
+RUN apk add coreutils binutils findutils grep
+
+ADD builder /builder
+
+EXPOSE 8001

buildermgr/api.go

@@ -39,10 +39,9 @@ type (
 	}
 
 	BuilderMgr struct {
-		fissionClient    *crd.FissionClient
+		fissionClient *crd.FissionClient
-		kubernetesClient *kubernetes.Clientset
+		storageSvcUrl string
-		storageSvcUrl    string
+		namespace     string
-		namespace        string
 	}
 )
 
@@ -53,14 +52,13 @@ func MakeBuilderMgr(fissionClient *crd.FissionClient,
 	envWatcher := makeEnvironmentWatcher(fissionClient, kubernetesClient, envBuilderNamespace)
 	go envWatcher.watchEnvironments()
 
-	pkgWatcher := makePackageWatcher(fissionClient, kubernetesClient, envBuilderNamespace, storageSvcUrl)
+	pkgWatcher := makePackageWatcher(fissionClient, envBuilderNamespace, storageSvcUrl)
 	go pkgWatcher.watchPackages()
 
 	return &BuilderMgr{
-		fissionClient:    fissionClient,
+		fissionClient: fissionClient,
-		kubernetesClient: kubernetesClient,
+		storageSvcUrl: storageSvcUrl,
-		storageSvcUrl:    storageSvcUrl,
+		namespace:     envBuilderNamespace,
-		namespace:        envBuilderNamespace,
 	}
 }
 
@@ -76,14 +74,23 @@ func (builderMgr *BuilderMgr) build(w http.ResponseWriter, r *http.Request) {
 	buildReq := BuildRequest{}
 	err = json.Unmarshal([]byte(body), &buildReq)
 	if err != nil {
-		e := fmt.Sprintf("invalid request body: %v", err)
+		e := fmt.Sprintf("Invalid request body: %v", err)
 		log.Println(e)
 		http.Error(w, e, 400)
 		return
 	}
 
-	buildLogs, err := buildPackage(builderMgr.fissionClient, builderMgr.kubernetesClient,
+	pkg, err := builderMgr.fissionClient.
-		builderMgr.namespace, builderMgr.storageSvcUrl, buildReq)
+		Packages(buildReq.Package.Namespace).
+		Get(buildReq.Package.Name)
+	if err != nil {
+		e := fmt.Sprintf("Error getting package CRD info: %v", err)
+		log.Println(e)
+		http.Error(w, e, 500)
+		return
+	}
+
+	buildLogs, err := buildPackage(builderMgr.fissionClient, builderMgr.namespace, builderMgr.storageSvcUrl, pkg)
 	if err != nil {
 		code, e := fission.GetHTTPError(err)
 		http.Error(w, e, code)

buildermgr/common.go

@@ -19,10 +19,10 @@ package buildermgr
 import (
 	"fmt"
 	"log"
+	"net/http"
 	"strings"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
 
 	"github.com/dchest/uniuri"
 	"github.com/fission/fission"
@@ -43,23 +43,14 @@ import (
 // 6. Update package status to succeed state
 // 7. Update package resource in package ref of functions that share the same package
 // *. Update package status to failed state,if any one of steps above failed
-func buildPackage(fissionClient *crd.FissionClient, kubernetesClient *kubernetes.Clientset,
+func buildPackage(fissionClient *crd.FissionClient, builderNamespace string,
-	builderNamespace string, storageSvcUrl string, buildReq BuildRequest) (buildLogs string, err error) {
+	storageSvcUrl string, pkg *crd.Package) (buildLogs string, err error) {
-
-	pkg, err := fissionClient.Packages(
-		buildReq.Package.Namespace).Get(buildReq.Package.Name)
-	if err != nil {
-		e := fmt.Sprintf("Error getting function CRD info: %v", err)
-		log.Println(e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
-		return e, fission.MakeError(500, e)
-	}
 
 	// Only do build for pending packages
 	if pkg.Status.BuildStatus != fission.BuildStatusPending {
 		e := "package is not in pending state"
 		log.Println(e)
-		return e, fission.MakeError(400, e)
+		return e, fission.MakeError(http.StatusBadRequest, e)
 	}
 
 	// update package status to running state, so that
@@ -76,7 +67,7 @@ func buildPackage(fissionClient *crd.FissionClient, kubernetesClient *kubernetes
 		e := fmt.Sprintf("Error setting package pending state: %v", err)
 		log.Println(e)
 		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
-		return e, fission.MakeError(500, e)
+		return e, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
 	env, err := fissionClient.Environments(metav1.NamespaceDefault).Get(pkg.Spec.Environment.Name)
@@ -84,7 +75,7 @@ func buildPackage(fissionClient *crd.FissionClient, kubernetesClient *kubernetes
 		e := fmt.Sprintf("Error getting environment CRD info: %v", err)
 		log.Println(e)
 		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
-		return e, fission.MakeError(500, e)
+		return e, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
 	svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, builderNamespace)
@@ -104,12 +95,17 @@ func buildPackage(fissionClient *crd.FissionClient, kubernetesClient *kubernetes
 		e := fmt.Sprintf("Error fetching source package: %v", err)
 		log.Println(e)
 		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
-		return e, fission.MakeError(500, e)
+		return e, fission.MakeError(http.StatusInternalServerError, e)
+	}
+
+	buildCmd := pkg.Spec.BuildCommand
+	if len(buildCmd) == 0 {
+		buildCmd = env.Spec.Builder.Command
 	}
 
 	pkgBuildReq := &builder.PackageBuildRequest{
 		SrcPkgFilename: srcPkgFilename,
-		BuildCommand:   pkg.Spec.BuildCommand,
+		BuildCommand:   buildCmd,
 	}
 
 	log.Printf("Start building with source package: %v", srcPkgFilename)
@@ -118,8 +114,13 @@ func buildPackage(fissionClient *crd.FissionClient, kubernetesClient *kubernetes
 	if err != nil {
 		e := fmt.Sprintf("Error building deployment package: %v", err)
 		log.Println(e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		var buildLogs string
-		return e, fission.MakeError(500, e)
+		if buildResp != nil {
+			buildLogs = buildResp.BuildLogs
+		}
+		buildLogs += fmt.Sprintf("%v\n", e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
+		return e, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
 	log.Printf("Build succeed, source package: %v, deployment package: %v", srcPkgFilename, buildResp.ArtifactFilename)
@@ -135,8 +136,9 @@ func buildPackage(fissionClient *crd.FissionClient, kubernetesClient *kubernetes
 	if err != nil {
 		e := fmt.Sprintf("Error uploading deployment package: %v", err)
 		log.Println(e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
-		return e, fission.MakeError(500, e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildResp.BuildLogs, nil)
+		return e, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
 	log.Printf("Start updating info of package: %v", pkg.Metadata.Name)
@@ -147,8 +149,9 @@ func buildPackage(fissionClient *crd.FissionClient, kubernetesClient *kubernetes
 	if err != nil {
 		e := fmt.Sprintf("Error creating deployment package CRD resource: %v", err)
 		log.Println(e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
-		return e, fission.MakeError(500, e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildResp.BuildLogs, nil)
+		return e, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
 	fnList, err := fissionClient.
@@ -156,8 +159,9 @@ func buildPackage(fissionClient *crd.FissionClient, kubernetesClient *kubernetes
 	if err != nil {
 		e := fmt.Sprintf("Error getting function list: %v", err)
 		log.Println(e)
-		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+		buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
-		return e, fission.MakeError(500, e)
+		updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildResp.BuildLogs, nil)
+		return e, fission.MakeError(http.StatusInternalServerError, e)
 	}
 
 	// A package may be used by multiple functions. Update
@@ -172,8 +176,9 @@ func buildPackage(fissionClient *crd.FissionClient, kubernetesClient *kubernetes
 			if err != nil {
 				e := fmt.Sprintf("Error updating function package resource version: %v", err)
 				log.Println(e)
-				updatePackage(fissionClient, pkg, fission.BuildStatusFailed, e, nil)
+				buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
-				return e, fission.MakeError(500, e)
+				updatePackage(fissionClient, pkg, fission.BuildStatusFailed, buildResp.BuildLogs, nil)
+				return e, fission.MakeError(http.StatusInternalServerError, e)
 			}
 		}
 	}

buildermgr/pkgwatcher.go

@@ -22,7 +22,6 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/watch"
-	"k8s.io/client-go/kubernetes"
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/crd"
@@ -31,31 +30,25 @@ import (
 type (
 	packageWatcher struct {
 		fissionClient    *crd.FissionClient
-		kubernetesClient *kubernetes.Clientset
 		builderNamespace string
 		storageSvcUrl    string
 	}
 )
 
 func makePackageWatcher(fissionClient *crd.FissionClient,
-	kubernetesClient *kubernetes.Clientset, builderNamespace string, storageSvcUrl string) *packageWatcher {
+	builderNamespace string, storageSvcUrl string) *packageWatcher {
 	pkgw := &packageWatcher{
 		fissionClient:    fissionClient,
-		kubernetesClient: kubernetesClient,
 		builderNamespace: builderNamespace,
 		storageSvcUrl:    storageSvcUrl,
 	}
 	return pkgw
 }
 
-func (pkgw *packageWatcher) build(pkgMetadata metav1.ObjectMeta) {
+func (pkgw *packageWatcher) build(pkg *crd.Package) {
-	buildReq := BuildRequest{
+	_, err := buildPackage(pkgw.fissionClient, pkgw.builderNamespace, pkgw.storageSvcUrl, pkg)
-		Package: pkgMetadata,
-	}
-	_, err := buildPackage(pkgw.fissionClient,
-		pkgw.kubernetesClient, pkgw.builderNamespace, pkgw.storageSvcUrl, buildReq)
 	if err != nil {
-		log.Printf("Error building package %v: %v", buildReq.Package.Name, err)
+		log.Printf("Error building package %v: %v", pkg.Metadata.Name, err)
 	}
 }
 
@@ -84,7 +77,7 @@ func (pkgw *packageWatcher) watchPackages() {
 
 			// only do build for packages in pending state
 			if pkg.Status.BuildStatus == fission.BuildStatusPending {
-				go pkgw.build(pkg.Metadata)
+				go pkgw.build(pkg)
 			}
 		}
 	}

charts/README.md

@@ -48,7 +48,6 @@ The following table lists the configurable parameters of the Fission chart and t
 | `routerPort`        | Fission Router Service Port                | `31314`                  |
 | `functionNamespace` | Namespace for Fission functions            | `fission-function`       |
 | `builderNamespace`  | Namespace for Fission environment builders | `fission-builder`        |
-| `openshift`         | RBAC for openshift                         | `false`                  |
 
 
 * Extra configuration for `fission-all`

charts/fission-all/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: fission-all
-version: 0.4.0
+version: 0.4.1
 description: Fission is a fast serverless framework for Kubernetes.
 keywords:
 - fission
@@ -12,4 +12,4 @@ maintainers:
   - name: Ta Ching Chen
     email: contact@tachingchen.com
 engine: gotpl
-appVersion: 0.4.0
+appVersion: 0.4.1

charts/fission-all/templates/NOTES.txt

@@ -1,13 +1,13 @@
 1. Install the client CLI.
 
 Mac:
-  $ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.0/fission-cli-osx && chmod +x fission && sudo mv fission /usr/local/bin/
+  $ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.1/fission-cli-osx && chmod +x fission && sudo mv fission /usr/local/bin/
 
 Linux:
-  $ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.0/fission-cli-linux && chmod +x fission && sudo mv fission /usr/local/bin/
+  $ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.1/fission-cli-linux && chmod +x fission && sudo mv fission /usr/local/bin/
 
 Windows:
-  For Windows, you can use the linux binary on WSL. Or you can download this windows executable: https://github.com/fission/fission/releases/download/0.4.0/fission-cli-windows.exe
+  For Windows, you can use the linux binary on WSL. Or you can download this windows executable: https://github.com/fission/fission/releases/download/0.4.1/fission-cli-windows.exe
 
 2. Set the FISSION_URL and FISSION_ROUTER environment variables.
 

charts/fission-all/templates/deployment.yaml

@@ -1,77 +1,3 @@
-{{ if .Values.openshift }}
-
-# For openshift
-
----
-apiVersion: v1
-kind: ProjectRequest
-metadata:
-  name: fission
-  labels:
-    name: fission
-
----
-apiVersion: v1
-kind: ProjectRequest
-metadata:
-  name: {{ .Values.functionNamespace }}
-  labels:
-    name: fission-function
-
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: fission-admin
-  namespace: {{ .Release.Namespace }}
-
----
-apiVersion: v1
-kind: ClusterRole
-metadata:
-  name: fission:fission-admin
-rules:
-- apiGroups:
-  - extensions
-  attributeRestrictions: null
-  resources:
-  - deployments
-  verbs:
-  - create
-  - get
-  - list
-  - update
-  - watch
-- apiGroups:
-  - ""
-  attributeRestrictions: null
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - update
-
----
-apiVersion: v1
-groupNames: null
-kind: RoleBinding
-metadata:
-  name: fission:fission-admin
-  namespace: {{ .Values.functionNamespace }}
-roleRef:
-  name: fission:fission-admin
-subjects:
-- kind: ServiceAccount
-  name: fission-admin
-  namespace: {{ .Release.Namespace }}
-userNames:
-- system:serviceaccount:fission:fission-admin
-
-{{ else }}
-
-# For all environments except openshift
-
 ---
 apiVersion: v1
 kind: Namespace
@@ -183,8 +109,6 @@ roleRef:
   name: cluster-admin
   apiGroup: rbac.authorization.k8s.io
 
-{{ end }}
-
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -226,16 +150,16 @@ spec:
         image: "{{ .Values.image }}:{{ .Values.imageTag }}"
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
-        args: ["--routerPort", "8888", "--poolmgrUrl", "http://poolmgr.{{ .Release.Namespace }}"]
+        args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"]
       serviceAccount: fission-svc
 
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: poolmgr
+  name: executor
   labels:
-    svc: poolmgr
+    svc: executor
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
 spec:
   type: ClusterIP
@@ -243,13 +167,13 @@ spec:
   - port: 80
     targetPort: 8888
   selector:
-    svc: poolmgr
+    svc: executor
 
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: poolmgr
+  name: executor
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
 spec:
@@ -257,14 +181,14 @@ spec:
   template:
     metadata:
       labels:
-        svc: poolmgr
+        svc: executor
     spec:
       containers:
-      - name: poolmgr
+      - name: executor
         image: "{{ .Values.image }}:{{ .Values.imageTag }}"
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
-        args: ["--poolmgrPort", "8888", "--namespace", "{{ .Values.functionNamespace }}", "--fission-namespace", "{{ .Release.Namespace }}"]
+        args: ["--executorPort", "8888", "--namespace", "{{ .Values.functionNamespace }}", "--fission-namespace", "{{ .Release.Namespace }}"]
         env:
         - name: FETCHER_IMAGE
           value: "{{ .Values.fetcherImage }}:{{ .Values.fetcherImageTag }}"
@@ -273,7 +197,6 @@ spec:
         - name: RUNTIME_IMAGE_PULL_POLICY
           value: "{{ .Values.pullPolicy }}"
       serviceAccount: fission-svc
-
 ---
 apiVersion: v1
 kind: Service

charts/fission-all/templates/pvc.yaml

@@ -7,17 +7,17 @@ metadata:
     app: fission-storage
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     release: "{{ .Release.Name }}"
-  annotations:
-  {{- if .Values.persistence.storageClass }}
-    volume.beta.kubernetes.io/storage-class: {{ .Values.persistence.storageClass | quote }}
-  {{- else }}
-    volume.alpha.kubernetes.io/storage-class: default
-  {{- end }}
 spec:
   accessModes:
     - {{ .Values.persistence.accessMode | quote }}
   resources:
     requests:
       storage: {{ .Values.persistence.size | quote }}
-
+  {{- if .Values.persistence.storageClass }}
+  {{- if (eq "-" .Values.persistence.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: {{ .Values.persistence.storageClass | quote }}
+  {{- end }}
+  {{- end }}
 {{- end }}

charts/fission-all/values.yaml

@@ -13,13 +13,13 @@ image: fission/fission-bundle
 pullPolicy: IfNotPresent
 
 ## Fission image version
-imageTag: 0.4.0
+imageTag: 0.4.1
 
 ## Fission fetcher repository
 fetcherImage: fission/fetcher
 
 ## Fission fetcher image version
-fetcherImageTag: 0.4.0
+fetcherImageTag: 0.4.1
 
 ## Port at which Fission controller service should be exposed
 controllerPort: 31313
@@ -38,9 +38,6 @@ functionNamespace: fission-function
 ## the release namespace)
 builderNamespace: fission-builder
 
-## Set up openshift RBAC rule
-openshift: false
-
 ## Logger config
 logger:
   influxdbAdmin: "admin"
@@ -56,10 +53,13 @@ nats:
 ## Persist data to a persistent volume.
 persistence:
   enabled: true
-  ## If defined, volume.beta.kubernetes.io/storage-class: <storageClass>
+  ## If defined, storageClassName: <storageClass>
-  ## Default: volume.alpha.kubernetes.io/storage-class: default
+  ## If set to "-", storageClassName: "", which disables dynamic provisioning
+  ## If undefined (the default) or set to null, no storageClassName spec is
+  ##   set, choosing the default provisioner. (gp2 on AWS, standard on
+  ##   GKE, AWS & OpenStack)
   ##
-  # storageClass:
+  # storageClass: "-"
   accessMode: ReadWriteOnce
   size: 8Gi
 

charts/fission-core/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: fission-core
-version: 0.4.0
+version: 0.4.1
 description: Fission is a fast serverless framework for Kubernetes.
 keywords:
 - fission
@@ -12,4 +12,4 @@ maintainers:
   - name: Ta Ching Chen
     email: contact@tachingchen.com
 engine: gotpl
-appVersion: 0.4.0
+appVersion: 0.4.1

charts/fission-core/templates/NOTES.txt

@@ -1,13 +1,13 @@
 1. Install the client CLI.
 
 Mac:
-  $ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.0/fission-cli-osx && chmod +x fission && sudo mv fission /usr/local/bin/
+  $ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.1/fission-cli-osx && chmod +x fission && sudo mv fission /usr/local/bin/
 
 Linux:
-  $ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.0/fission-cli-linux && chmod +x fission && sudo mv fission /usr/local/bin/
+  $ curl -Lo fission https://github.com/fission/fission/releases/download/0.4.1/fission-cli-linux && chmod +x fission && sudo mv fission /usr/local/bin/
 
 Windows:
-  For Windows, you can use the linux binary on WSL. Or you can download this windows executable: https://github.com/fission/fission/releases/download/0.4.0/fission-cli-windows.exe
+  For Windows, you can use the linux binary on WSL. Or you can download this windows executable: https://github.com/fission/fission/releases/download/0.4.1/fission-cli-windows.exe
 
 2. Set the FISSION_URL and FISSION_ROUTER environment variables.
 

charts/fission-core/templates/deployment.yaml

@@ -1,77 +1,3 @@
-{{ if .Values.openshift }}
-
-# For openshift
-
----
-apiVersion: v1
-kind: ProjectRequest
-metadata:
-  name: fission
-  labels:
-    name: fission
-
----
-apiVersion: v1
-kind: ProjectRequest
-metadata:
-  name: {{ .Values.functionNamespace }}
-  labels:
-    name: fission-function
-
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: fission-admin
-  namespace: {{ .Release.Namespace }}
-
----
-apiVersion: v1
-kind: ClusterRole
-metadata:
-  name: fission:fission-admin
-rules:
-- apiGroups:
-  - extensions
-  attributeRestrictions: null
-  resources:
-  - deployments
-  verbs:
-  - create
-  - get
-  - list
-  - update
-  - watch
-- apiGroups:
-  - ""
-  attributeRestrictions: null
-  resources:
-  - pods
-  verbs:
-  - get
-  - list
-  - update
-
----
-apiVersion: v1
-groupNames: null
-kind: RoleBinding
-metadata:
-  name: fission:fission-admin
-  namespace: {{ .Values.functionNamespace }}
-roleRef:
-  name: fission:fission-admin
-subjects:
-- kind: ServiceAccount
-  name: fission-admin
-  namespace: {{ .Release.Namespace }}
-userNames:
-- system:serviceaccount:fission:fission-admin
-
-{{ else }}
-
-# For all environments except openshift
-
 ---
 apiVersion: v1
 kind: Namespace
@@ -183,8 +109,6 @@ roleRef:
   name: cluster-admin
   apiGroup: rbac.authorization.k8s.io
 
-{{ end }}
-
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
@@ -226,16 +150,16 @@ spec:
         image: "{{ .Values.image }}:{{ .Values.imageTag }}"
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
-        args: ["--routerPort", "8888", "--poolmgrUrl", "http://poolmgr.{{ .Release.Namespace }}"]
+        args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"]
       serviceAccount: fission-svc
 
 ---
 apiVersion: v1
 kind: Service
 metadata:
-  name: poolmgr
+  name: executor
   labels:
-    svc: poolmgr
+    svc: executor
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
 spec:
   type: ClusterIP
@@ -243,13 +167,13 @@ spec:
   - port: 80
     targetPort: 8888
   selector:
-    svc: poolmgr
+    svc: executor
 
 ---
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: poolmgr
+  name: executor
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
 spec:
@@ -257,14 +181,14 @@ spec:
   template:
     metadata:
       labels:
-        svc: poolmgr
+        svc: executor
     spec:
       containers:
-      - name: poolmgr
+      - name: executor
         image: "{{ .Values.image }}:{{ .Values.imageTag }}"
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
-        args: ["--poolmgrPort", "8888", "--namespace", "{{ .Values.functionNamespace }}", "--fission-namespace", "{{ .Release.Namespace }}"]
+        args: ["--executorPort", "8888", "--namespace", "{{ .Values.functionNamespace }}", "--fission-namespace", "{{ .Release.Namespace }}"]
         env:
         - name: FETCHER_IMAGE
           value: "{{ .Values.fetcherImage }}:{{ .Values.fetcherImageTag }}"

charts/fission-core/templates/pvc.yaml

@@ -7,17 +7,17 @@ metadata:
     app: fission-storage
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     release: "{{ .Release.Name }}"
-  annotations:
-  {{- if .Values.persistence.storageClass }}
-    volume.beta.kubernetes.io/storage-class: {{ .Values.persistence.storageClass | quote }}
-  {{- else }}
-    volume.alpha.kubernetes.io/storage-class: default
-  {{- end }}
 spec:
   accessModes:
     - {{ .Values.persistence.accessMode | quote }}
   resources:
     requests:
       storage: {{ .Values.persistence.size | quote }}
-
+  {{- if .Values.persistence.storageClass }}
+  {{- if (eq "-" .Values.persistence.storageClass) }}
+  storageClassName: ""
+  {{- else }}
+  storageClassName: {{ .Values.persistence.storageClass | quote }}
+  {{- end }}
+  {{- end }}
 {{- end }}

charts/fission-core/values.yaml

@@ -10,7 +10,7 @@ serviceType: LoadBalancer
 image: fission/fission-bundle
 
 ## Fission image version
-imageTag: 0.4.0
+imageTag: 0.4.1
 
 ## Image pull policy
 pullPolicy: IfNotPresent
@@ -19,7 +19,7 @@ pullPolicy: IfNotPresent
 fetcherImage: fission/fetcher
 
 ## Fission fetcher image version
-fetcherImageTag: 0.4.0
+fetcherImageTag: 0.4.1
 
 ## Port at which Fission controller service should be exposed
 controllerPort: 31313
@@ -35,16 +35,16 @@ functionNamespace: fission-function
 ## the release namespace)
 builderNamespace: fission-builder
 
-## Set up openshift RBAC rule
-openshift: false
-  
 ## Persist data to a persistent volume.
 persistence:
   enabled: true
-  ## If defined, volume.beta.kubernetes.io/storage-class: <storageClass>
+  ## If defined, storageClassName: <storageClass>
-  ## Default: volume.alpha.kubernetes.io/storage-class: default
+  ## If set to "-", storageClassName: "", which disables dynamic provisioning
+  ## If undefined (the default) or set to null, no storageClassName spec is
+  ##   set, choosing the default provisioner. (gp2 on AWS, standard on
+  ##   GKE, AWS & OpenStack)
   ##
-  # storageClass:
+  # storageClass: "-"
   accessMode: ReadWriteOnce
   size: 8Gi
 

controller/api.go

@@ -120,7 +120,7 @@ func (api *API) getLogDBConfig(dbType string) logDBConfig {
 
 func (api *API) HomeHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	fmt.Fprintf(w, "{\"message\": \"Fission API\", \"version\": \"0.4.0\"}\n")
+	fmt.Fprintf(w, "{\"message\": \"Fission API\", \"version\": \"0.4.1\"}\n")
 }
 
 func (api *API) ApiVersionMismatchHandler(w http.ResponseWriter, r *http.Request) {

controller/httpTriggerApi.go

@@ -51,9 +51,9 @@ func (a *API) checkHTTPTriggerDuplicates(t *crd.HTTPTrigger) error {
 		return err
 	}
 	for _, ht := range triggers.Items {
-		if ht.Spec.RelativeURL == t.Spec.RelativeURL && ht.Spec.Method == t.Spec.Method {
+		if ht.Spec.RelativeURL == t.Spec.RelativeURL && ht.Spec.Method == t.Spec.Method && ht.Spec.Host == t.Spec.Host {
 			return fission.MakeError(fission.ErrorNameExists,
-				fmt.Sprintf("HTTPTrigger with same URL & method already exists (%v)",
+				fmt.Sprintf("HTTPTrigger with same Host, URL & method already exists (%v)",
 					ht.Metadata.Name))
 		}
 	}

demos/util.sh

@@ -37,10 +37,6 @@ function maybe_first_prompt() {
     fi
 }
 
-# After a `run` this variable will hold the stdout of the command that was run.
-# If the command was interactive, this will likely be garbage.
-#DEMO_RUN_STDOUT=""
-
 function run() {
     maybe_first_prompt
     rate=25
@@ -52,27 +48,11 @@ function run() {
       sleep 0.5
     fi
     $1
-    echo; echo -n ">"
+    echo
+    echo -n ">"
     read -s
     echo -e "\b "
-#    OFILE="$(mktemp -t $(basename $0).XXXXXX)"
-#    script -q -t0 "$OFILE" $1 # [removed -e,-c options -soam]
-#    r=$?
-#    read -d '' -t "${timeout}" -n 10000 # clear stdin
-#    prompt
-#    if [ -z "$DEMO_AUTO_RUN" ]; then
-#      read -s
-#    fi
-#    DEMO_RUN_STDOUT="$(tail -n +2 $OFILE | sed 's/\r//g')"
-#    return $r
-}
-
-function relative() {
-    for arg; do
-        echo "$(realpath $(dirname $(which $0)))/$arg" | sed "s|$(realpath $(pwd))|.|"
-    done
 }
 
 SSH_NODE=$(kubectl get nodes | tail -1 | cut -f1 -d' ')
 
-#trap "echo" EXIT

environments/fetcher/client/client.go

@@ -4,12 +4,14 @@ import (
 	"bytes"
 	"encoding/json"
 	"io/ioutil"
+	"log"
+	"net"
 	"net/http"
-	//"time"
+	"net/url"
+	"time"
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/environments/fetcher"
-	//"github.com/fission/fission/router"
 )
 
 type (
@@ -30,21 +32,35 @@ func (c *Client) Fetch(fr *fetcher.FetchRequest) error {
 		return err
 	}
 
-	// client := http.Client{
+	maxRetries := 20
-	// 	Transport: router.MakeRetryingRoundTripper(10, 50*time.Millisecond),
+	var resp *http.Response
-	// }
 
-	resp, err := http.Post(c.url, "application/json", bytes.NewReader(body))
+	for i := 0; i < maxRetries; i++ {
-	if err != nil {
+		resp, err = http.Post(c.url, "application/json", bytes.NewReader(body))
-		return err
-	}
-	defer resp.Body.Close()
 
-	if resp.StatusCode != 200 {
+		if err == nil && resp.StatusCode == 200 {
-		return fission.MakeErrorFromHTTP(resp)
+			defer resp.Body.Close()
+			return nil
+		}
+
+		// Only retry for the specific case of a connection error.
+		if urlErr, ok := err.(*url.Error); ok {
+			if netErr, ok := urlErr.Err.(*net.OpError); ok {
+				if netErr.Op == "dial" {
+					if i < maxRetries-1 {
+						time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
+						continue
+					}
+				}
+			}
+		}
 	}
 
-	return nil
+	if err == nil {
+		err = fission.MakeErrorFromHTTP(resp)
+	}
+	log.Printf("Failed to fetch: %v", err)
+	return err
 }
 
 func (c *Client) Upload(fr *fetcher.UploadRequest) (*fetcher.UploadResponse, error) {

environments/fetcher/fetcher.go

@@ -78,12 +78,15 @@ func downloadUrl(url string, localPath string) error {
 	}
 	defer resp.Body.Close()
 
-	body, err := ioutil.ReadAll(resp.Body)
+	w, err := os.Create(localPath)
 	if err != nil {
 		return err
 	}
-
+	_, err = io.Copy(w, resp.Body)
-	err = ioutil.WriteFile(localPath, body, 0600)
+	if err != nil {
+		return err
+	}
+	err = os.Chmod(localPath, 0600)
 	if err != nil {
 		return err
 	}
@@ -327,7 +330,8 @@ func (fetcher *Fetcher) rename(src string, dst string) error {
 	return nil
 }
 
-// archive is a function that zips directory into a zip file
+// archive zips the contents of directory at src into a new zip file
+// at dst (note that the contents are zipped, not the directory itself).
 func (fetcher *Fetcher) archive(src string, dst string) error {
 	var files []string
 	target, err := os.Stat(src)

environments/go/Dockerfile

@@ -1,4 +1,6 @@
-FROM golang:1.8.1
+ARG GO_VERSION=1.9.2
+
+FROM golang:${GO_VERSION}
 
 ENV GOPATH /usr
 ENV APP	   ${GOPATH}/src/github.com/fission/fission/environments/go

environments/go/builder/Dockerfile

@@ -1,5 +1,14 @@
-FROM golang:1.8.1
+ARG BUILDER_IMAGE=fission/builder
+ARG GO_VERSION=1.9.2
+
+FROM ${BUILDER_IMAGE}
+
+FROM golang:${GO_VERSION}
+
+COPY --from=0 /builder /builder
 
 ENV GOPATH /usr
 RUN go get github.com/fission/fission
 
+ADD build.sh /usr/local/bin/build
+

environments/go/builder/build.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+cd ${SRC_PKG}
+go build -buildmode=plugin -i -o ${DEPLOY_PKG} .

environments/go/builder/go-function-build

@@ -1,11 +0,0 @@
-#!/bin/sh
-
-if [ $# -eq 0 ]
-then
-    echo "No arguments, building current dir"
-    filename='.'
-else
-    filename=$1
-fi
-
-docker run --rm -v $(pwd):/build -w /build fission/go-builder:1.8.1 go build -buildmode=plugin -o function.so $filename && echo "Compiled plugin: function.so. Use 'fission function create' to upload to fission."

environments/go/server.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
+	"path/filepath"
 	"plugin"
 
 	"github.com/fission/fission/environments/go/context"
@@ -13,14 +16,52 @@ const (
 	CODE_PATH = "/userfunc/user"
 )
 
+type (
+	FunctionLoadRequest struct {
+		// FilePath is an absolute filesystem path to the
+		// function. What exactly is stored here is
+		// env-specific. Optional.
+		FilePath string `json:"filepath"`
+
+		// FunctionName has an environment-specific meaning;
+		// usually, it defines a function within a module
+		// containing multiple functions. Optional; default is
+		// environment-specific.
+		FunctionName string `json:"functionName"`
+
+		// URL to expose this function at. Optional; defaults
+		// to "/".
+		URL string `json:"url"`
+	}
+)
+
 var userFunc http.HandlerFunc
 
-func loadPlugin() http.HandlerFunc {
+func loadPlugin(codePath, entrypoint string) http.HandlerFunc {
-	p, err := plugin.Open(CODE_PATH)
+
+	// if codepath's a directory, load the file inside it
+	info, err := os.Stat(codePath)
 	if err != nil {
 		panic(err)
 	}
-	sym, err := p.Lookup("Handler")
+	if info.IsDir() {
+		files, err := ioutil.ReadDir(codePath)
+		if err != nil {
+			panic(err)
+		}
+		if len(files) == 0 {
+			panic("No files to load")
+		}
+		fi := files[0]
+		codePath = filepath.Join(codePath, fi.Name())
+	}
+
+	fmt.Printf("loading plugin from %v\n", codePath)
+	p, err := plugin.Open(codePath)
+	if err != nil {
+		panic(err)
+	}
+	sym, err := p.Lookup(entrypoint)
 	if err != nil {
 		panic("Entry point not found")
 	}
@@ -44,7 +85,7 @@ func loadPlugin() http.HandlerFunc {
 
 func specializeHandler(w http.ResponseWriter, r *http.Request) {
 	if userFunc != nil {
-		w.WriteHeader(400)
+		w.WriteHeader(http.StatusBadRequest)
 		w.Write([]byte("Not a generic container"))
 		return
 	}
@@ -61,12 +102,48 @@ func specializeHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	fmt.Println("Specializing ...")
-	userFunc = loadPlugin()
+	userFunc = loadPlugin(CODE_PATH, "Handler")
+	fmt.Println("Done")
+}
+
+func specializeHandlerV2(w http.ResponseWriter, r *http.Request) {
+	if userFunc != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte("Not a generic container"))
+		return
+	}
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+	var loadreq FunctionLoadRequest
+	err = json.Unmarshal(body, &loadreq)
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	_, err = os.Stat(loadreq.FilePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(CODE_PATH + ": not found"))
+			return
+		} else {
+			panic(err)
+		}
+	}
+
+	fmt.Println("Specializing ...")
+	userFunc = loadPlugin(loadreq.FilePath, loadreq.FunctionName)
 	fmt.Println("Done")
 }
 
 func main() {
 	http.HandleFunc("/specialize", specializeHandler)
+	http.HandleFunc("/v2/specialize", specializeHandlerV2)
 
 	// Generic route -- all http requests go to the user function.
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {

environments/python/builder/Dockerfile

@@ -1,4 +1,5 @@
-FROM alpine:3.5
+ARG BUILDER_IMAGE=fission/builder:latest
+FROM ${BUILDER_IMAGE}
 
 RUN apk update
 RUN apk add --no-cache python3 python3-dev build-base
@@ -6,6 +7,5 @@ RUN pip3 install --upgrade pip
 RUN rm -r /root/.cache
 
 ADD defaultBuildCmd /usr/local/bin/build
-ADD builder /builder
 
 EXPOSE 8001

environments/python/server.py

@@ -95,4 +95,4 @@ def setup_logger(loglevel):
 #
 setup_logger(logging.DEBUG)
 app.logger.info("Starting server")
-app.run(host='0.0.0.0', port='8888')
+app.run(host='0.0.0.0', port=8888)

executor/api.go

@@ -0,0 +1,108 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package executor
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/gorilla/handlers"
+	"github.com/gorilla/mux"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission"
+)
+
+func (executor *Executor) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, "Failed to read request", 500)
+		return
+	}
+
+	// get function metadata
+	m := metav1.ObjectMeta{}
+	err = json.Unmarshal(body, &m)
+	if err != nil {
+		http.Error(w, "Failed to parse request", 400)
+		return
+	}
+
+	serviceName, err := executor.getServiceForFunction(&m)
+	if err != nil {
+		code, msg := fission.GetHTTPError(err)
+		log.Printf("Error: %v: %v", code, msg)
+		http.Error(w, msg, code)
+		return
+	}
+
+	w.Write([]byte(serviceName))
+}
+
+func (executor *Executor) getServiceForFunction(m *metav1.ObjectMeta) (string, error) {
+	// Check function -> svc cache
+	log.Printf("[%v] Checking for cached function service", m.Name)
+	fsvc, err := executor.fsCache.GetByFunction(m)
+	if err == nil {
+		// Cached, return svc address
+		return fsvc.Address, nil
+	}
+
+	respChan := make(chan *createFuncServiceResponse)
+	executor.requestChan <- &createFuncServiceRequest{
+		funcMeta: m,
+		respChan: respChan,
+	}
+	resp := <-respChan
+	if resp.err != nil {
+		return "", resp.err
+	}
+	return resp.funcSvc.Address, resp.err
+}
+
+// find funcSvc and update its atime
+func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, "Failed to read request", 500)
+		return
+	}
+	svcName := string(body)
+	svcHost := strings.TrimPrefix(svcName, "http://")
+
+	err = executor.fsCache.TouchByAddress(svcHost)
+	if err != nil {
+		log.Printf("funcSvc tap error: %v", err)
+		http.Error(w, "Not found", 404)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+func (executor *Executor) Serve(port int) {
+	r := mux.NewRouter()
+	r.HandleFunc("/v2/getServiceForFunction", executor.getServiceForFunctionApi).Methods("POST")
+	r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST")
+	address := fmt.Sprintf(":%v", port)
+	log.Printf("starting executor at port %v", port)
+	log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r)))
+}

executor/client/client.go

@@ -32,14 +32,14 @@ import (
 )
 
 type Client struct {
-	poolmgrUrl  string
+	executorUrl string
 	tappedByUrl map[string]bool
 	requestChan chan string
 }
 
-func MakeClient(poolmgrUrl string) *Client {
+func MakeClient(executorUrl string) *Client {
 	c := &Client{
-		poolmgrUrl:  strings.TrimSuffix(poolmgrUrl, "/"),
+		executorUrl: strings.TrimSuffix(executorUrl, "/"),
 		tappedByUrl: make(map[string]bool),
 		requestChan: make(chan string),
 	}
@@ -48,14 +48,14 @@ func MakeClient(poolmgrUrl string) *Client {
 }
 
 func (c *Client) GetServiceForFunction(metadata *metav1.ObjectMeta) (string, error) {
-	poolmgrUrl := c.poolmgrUrl + "/v2/getServiceForFunction"
+	executorUrl := c.executorUrl + "/v2/getServiceForFunction"
 
 	body, err := json.Marshal(metadata)
 	if err != nil {
 		return "", err
 	}
 
-	resp, err := http.Post(poolmgrUrl, "application/json", bytes.NewReader(body))
+	resp, err := http.Post(executorUrl, "application/json", bytes.NewReader(body))
 	if err != nil {
 		return "", err
 	}
@@ -100,9 +100,9 @@ func (c *Client) TapService(serviceUrl *url.URL) {
 }
 
 func (c *Client) _tapService(serviceUrlStr string) error {
-	poolmgrUrl := c.poolmgrUrl + "/v2/tapService"
+	executorUrl := c.executorUrl + "/v2/tapService"
 
-	resp, err := http.Post(poolmgrUrl, "application/octet-stream", bytes.NewReader([]byte(serviceUrlStr)))
+	resp, err := http.Post(executorUrl, "application/octet-stream", bytes.NewReader([]byte(serviceUrlStr)))
 	if err != nil {
 		return err
 	}

executor/deploymgr/newdeploy.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deploymgr
+
+func GetFuncSvc() (*funcSvc, error) {
+
+	return nil, nil
+}

executor/executor.go

@@ -0,0 +1,195 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package executor
+
+import (
+	"log"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/dchest/uniuri"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission/cache"
+	"github.com/fission/fission/crd"
+	"github.com/fission/fission/executor/fscache"
+	"github.com/fission/fission/executor/poolmgr"
+)
+
+type (
+	Executor struct {
+		gpm           *poolmgr.GenericPoolManager
+		functionEnv   *cache.Cache
+		fissionClient *crd.FissionClient
+		fsCache       *fscache.FunctionServiceCache
+
+		requestChan chan *createFuncServiceRequest
+		fsCreateWg  map[string]*sync.WaitGroup
+	}
+	createFuncServiceRequest struct {
+		funcMeta *metav1.ObjectMeta
+		respChan chan *createFuncServiceResponse
+	}
+
+	createFuncServiceResponse struct {
+		funcSvc *fscache.FuncSvc
+		err     error
+	}
+)
+
+func MakeExecutor(gpm *poolmgr.GenericPoolManager, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor {
+	executor := &Executor{
+		gpm:           gpm,
+		functionEnv:   cache.MakeCache(10*time.Second, 0),
+		fissionClient: fissionClient,
+		fsCache:       fsCache,
+
+		requestChan: make(chan *createFuncServiceRequest),
+		fsCreateWg:  make(map[string]*sync.WaitGroup),
+	}
+	go executor.serveCreateFuncServices()
+	return executor
+}
+
+// All non-cached function service requests go through this goroutine
+// serially. It parallelizes requests for different functions, and
+// ensures that for a given function, only one request causes a pod to
+// get specialized. In other words, it ensures that when there's an
+// ongoing request for a certain function, all other requests wait for
+// that request to complete.
+func (executor *Executor) serveCreateFuncServices() {
+	for {
+		req := <-executor.requestChan
+		m := req.funcMeta
+
+		// Cache miss -- is this first one to request the func?
+		wg, found := executor.fsCreateWg[crd.CacheKey(m)]
+		if !found {
+			// create a waitgroup for other requests for
+			// the same function to wait on
+			wg := &sync.WaitGroup{}
+			wg.Add(1)
+			executor.fsCreateWg[crd.CacheKey(m)] = wg
+
+			// launch a goroutine for each request, to parallelize
+			// the specialization of different functions
+			go func() {
+				fsvc, err := executor.createServiceForFunction(m)
+				req.respChan <- &createFuncServiceResponse{
+					funcSvc: fsvc,
+					err:     err,
+				}
+				delete(executor.fsCreateWg, crd.CacheKey(m))
+				wg.Done()
+			}()
+		} else {
+			// There's an existing request for this function, wait for it to finish
+			go func() {
+				log.Printf("Waiting for concurrent request for the same function: %v", m)
+				wg.Wait()
+
+				// get the function service from the cache
+				fsvc, err := executor.fsCache.GetByFunction(m)
+				req.respChan <- &createFuncServiceResponse{
+					funcSvc: fsvc,
+					err:     err,
+				}
+			}()
+		}
+	}
+}
+
+func (executor *Executor) createServiceForFunction(m *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
+	log.Printf("[%v] No cached function service found, creating one", m.Name)
+
+	env, err := executor.getFunctionEnv(m)
+	if err != nil {
+		return nil, err
+	}
+	// Appropriate backend handles the service creation
+	backend := os.Getenv("EXECUTOR_BACKEND")
+	switch backend {
+	case "DEPLOY":
+		return nil, nil
+	default:
+		pool, err := executor.gpm.GetPool(env)
+		if err != nil {
+			return nil, err
+		}
+		// from GenericPool -> get one function container
+		// (this also adds to the cache)
+		log.Printf("[%v] getting function service from pool", m.Name)
+		fsvc, err := pool.GetFuncSvc(m)
+		if err != nil {
+			return nil, err
+		}
+		return fsvc, nil
+	}
+}
+
+func (executor *Executor) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment, error) {
+	var env *crd.Environment
+
+	// Cached ?
+	result, err := executor.functionEnv.Get(crd.CacheKey(m))
+	if err == nil {
+		env = result.(*crd.Environment)
+		return env, nil
+	}
+
+	// Cache miss -- get func from controller
+	f, err := executor.fissionClient.Functions(m.Namespace).Get(m.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get env from metadata
+	log.Printf("[%v] getting env", m)
+	env, err = executor.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	// cache for future lookups
+	executor.functionEnv.Set(crd.CacheKey(m), env)
+
+	return env, nil
+}
+
+// StartExecutor Starts executor and the backend components that executor uses such as Poolmgr,
+// deploymgr and potential future backends
+func StartExecutor(fissionNamespace string, functionNamespace string, port int) error {
+	fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
+	if err != nil {
+		log.Printf("Failed to get kubernetes client: %v", err)
+		return err
+	}
+
+	instanceID := uniuri.NewLen(8)
+	poolmgr.CleanupOldPoolmgrResources(kubernetesClient, functionNamespace, instanceID)
+
+	fsCache := fscache.MakeFunctionServiceCache()
+	gpm := poolmgr.MakeGenericPoolManager(
+		fissionClient, kubernetesClient, fissionNamespace,
+		functionNamespace, fsCache, instanceID)
+
+	api := MakeExecutor(gpm, fissionClient, fsCache)
+	go api.Serve(port)
+
+	return nil
+}

executor/executor_test.go

@@ -20,7 +20,7 @@
 // Here's how I run this on my setup, with minikube:
 // TEST_SPECIALIZE_URL=http://192.168.99.100:30002/specialize TEST_FETCHER_URL=http://192.168.99.100:30001 FETCHER_IMAGE=minikube/fetcher:testing KUBECONFIG=/Users/soam/.kube/config go test -v .
 
-package poolmgr
+package executor
 
 import (
 	"fmt"
@@ -40,7 +40,7 @@ import (
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/crd"
-	"github.com/fission/fission/poolmgr/client"
+	"github.com/fission/fission/executor/client"
 )
 
 // return the number of pods in the given namespace matching the given labels
@@ -104,7 +104,7 @@ func httpGet(url string) string {
 	return string(body)
 }
 
-func TestPoolmgr(t *testing.T) {
+func TestExecutor(t *testing.T) {
 	// run in a random namespace so we can have concurrent tests
 	// on a given cluster
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -160,7 +160,7 @@ func TestPoolmgr(t *testing.T) {
 
 	// create poolmgr
 	port := 9999
-	err = StartPoolmgr(fissionNs, functionNs, port)
+	err = StartExecutor(fissionNs, functionNs, port)
 	if err != nil {
 		log.Panicf("failed to start poolmgr: %v", err)
 	}

executor/fscache/functionServiceCache.go

@@ -0,0 +1,230 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fscache
+
+import (
+	"log"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/pkg/api"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/cache"
+	"github.com/fission/fission/crd"
+)
+
+type fscRequestType int
+type backendType int
+
+const (
+	TOUCH fscRequestType = iota
+	LISTOLD
+	LOG
+)
+
+const (
+	POOLMGR backendType = iota
+	NEWDEPLOY
+)
+
+type (
+	FuncSvc struct {
+		Function          *metav1.ObjectMeta    // function this pod/service is for
+		Environment       *crd.Environment      // function's environment
+		Address           string                // Host:Port or IP:Port that the function's service can be reached at.
+		KubernetesObjects []api.ObjectReference // Kubernetes Objects (within the function namespace)
+		Backend           backendType
+
+		Ctime time.Time
+		Atime time.Time
+	}
+
+	FunctionServiceCache struct {
+		byFunction *cache.Cache // function-key -> funcSvc  : map[string]*funcSvc
+		byAddress  *cache.Cache // address      -> function : map[string]metav1.ObjectMeta
+
+		requestChannel chan *fscRequest
+	}
+	fscRequest struct {
+		requestType       fscRequestType
+		address           string
+		kubernetesObjects []api.ObjectReference
+		age               time.Duration
+		env               *metav1.ObjectMeta // used for ListOld
+		responseChannel   chan *fscResponse
+	}
+	fscResponse struct {
+		objects []*FuncSvc
+		deleted bool
+		error
+	}
+)
+
+func MakeFunctionServiceCache() *FunctionServiceCache {
+	fsc := &FunctionServiceCache{
+		byFunction:     cache.MakeCache(0, 0),
+		byAddress:      cache.MakeCache(0, 0),
+		requestChannel: make(chan *fscRequest),
+	}
+	go fsc.service()
+	return fsc
+}
+
+func (fsc *FunctionServiceCache) service() {
+	for {
+		req := <-fsc.requestChannel
+		resp := &fscResponse{}
+		switch req.requestType {
+		case TOUCH:
+			// update atime for this function svc
+			resp.error = fsc._touchByAddress(req.address)
+		case LISTOLD:
+			// get svcs idle for > req.age
+			fscs := fsc.byFunction.Copy()
+			funcObjects := make([]*FuncSvc, 0)
+			for _, funcSvc := range fscs {
+				fsvc := funcSvc.(*FuncSvc)
+				if fsvc.Environment.Metadata.UID == req.env.UID &&
+					time.Now().Sub(fsvc.Atime) > req.age {
+					funcObjects = append(funcObjects, fsvc)
+				}
+			}
+			resp.objects = funcObjects
+		case LOG:
+			funcCopy := fsc.byFunction.Copy()
+			log.Printf("Cache has %v entries", len(funcCopy))
+			for key, fsvcI := range funcCopy {
+				fsvc := fsvcI.(*FuncSvc)
+				for _, kubeObj := range fsvc.KubernetesObjects {
+					log.Printf("%v\t%v\t%v", key, kubeObj.Kind, kubeObj.Name)
+				}
+			}
+		}
+		req.responseChannel <- resp
+	}
+}
+
+func (fsc *FunctionServiceCache) GetByFunction(m *metav1.ObjectMeta) (*FuncSvc, error) {
+	key := crd.CacheKey(m)
+
+	fsvcI, err := fsc.byFunction.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	// update atime
+	fsvc := fsvcI.(*FuncSvc)
+	fsvc.Atime = time.Now()
+
+	fsvcCopy := *fsvc
+	return &fsvcCopy, nil
+}
+
+// TODO: error should be second return
+func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (error, *FuncSvc) {
+	err, existing := fsc.byFunction.Set(crd.CacheKey(fsvc.Function), &fsvc)
+	if err != nil {
+		if existing != nil {
+			f := existing.(*FuncSvc)
+			err2 := fsc.TouchByAddress(f.Address)
+			if err2 != nil {
+				return err2, nil
+			}
+			fCopy := *f
+			return err, &fCopy
+		}
+		return err, nil
+	}
+	now := time.Now()
+	fsvc.Ctime = now
+	fsvc.Atime = now
+
+	// Add to byAddress cache. Ignore NameExists errors
+	// because of multiple-specialization. See issue #331.
+	err, _ = fsc.byAddress.Set(fsvc.Address, *fsvc.Function)
+	if err != nil {
+		if fe, ok := err.(fission.Error); ok {
+			if fe.Code == fission.ErrorNameExists {
+				err = nil
+			}
+		}
+		log.Printf("error caching fsvc: %v", err)
+		return err, nil
+	}
+	return nil, nil
+}
+
+func (fsc *FunctionServiceCache) TouchByAddress(address string) error {
+	responseChannel := make(chan *fscResponse)
+	fsc.requestChannel <- &fscRequest{
+		requestType:     TOUCH,
+		address:         address,
+		responseChannel: responseChannel,
+	}
+	resp := <-responseChannel
+	return resp.error
+}
+
+func (fsc *FunctionServiceCache) _touchByAddress(address string) error {
+	mI, err := fsc.byAddress.Get(address)
+	if err != nil {
+		return err
+	}
+	m := mI.(metav1.ObjectMeta)
+	fsvcI, err := fsc.byFunction.Get(crd.CacheKey(&m))
+	if err != nil {
+		return err
+	}
+	fsvc := fsvcI.(*FuncSvc)
+	fsvc.Atime = time.Now()
+	return nil
+}
+
+func (fsc *FunctionServiceCache) DeleteOld(fsvc *FuncSvc, minAge time.Duration) (bool, error) {
+	if time.Now().Sub(fsvc.Atime) < minAge {
+		return false, nil
+	}
+
+	fsc.byFunction.Delete(crd.CacheKey(fsvc.Function))
+	fsc.byAddress.Delete(fsvc.Address)
+
+	return true, nil
+}
+
+func (fsc *FunctionServiceCache) ListOld(env *metav1.ObjectMeta, age time.Duration) ([]*FuncSvc, error) {
+	responseChannel := make(chan *fscResponse)
+	fsc.requestChannel <- &fscRequest{
+		requestType:     LISTOLD,
+		age:             age,
+		env:             env,
+		responseChannel: responseChannel,
+	}
+	resp := <-responseChannel
+	return resp.objects, resp.error
+}
+
+func (fsc *FunctionServiceCache) Log() {
+	log.Printf("--- FunctionService Cache Contents")
+	responseChannel := make(chan *fscResponse)
+	fsc.requestChannel <- &fscRequest{
+		requestType:     LOG,
+		responseChannel: responseChannel,
+	}
+	<-responseChannel
+	log.Printf("--- FunctionService Cache Contents End")
+}

executor/fscache/functionServiceCache_test.go

@@ -1,4 +1,4 @@
-package poolmgr
+package fscache
 
 import (
 	"log"
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/pkg/api"
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/crd"
@@ -17,15 +18,30 @@ func TestFunctionServiceCache(t *testing.T) {
 		log.Panicf("error creating cache")
 	}
 
-	var fsvc *funcSvc
+	var fsvc *FuncSvc
 	now := time.Now()
 
-	fsvc = &funcSvc{
+	objects := []api.ObjectReference{
-		function: &metav1.ObjectMeta{
+		{
+			Kind:       "pod",
+			Name:       "xxx",
+			APIVersion: "v1",
+			Namespace:  "fission-function",
+		},
+		{
+			Kind:       "pod",
+			Name:       "xxx2",
+			APIVersion: "v1",
+			Namespace:  "fission-function",
+		},
+	}
+
+	fsvc = &FuncSvc{
+		Function: &metav1.ObjectMeta{
 			Name: "foo",
 			UID:  "1212",
 		},
-		environment: &crd.Environment{
+		Environment: &crd.Environment{
 			Metadata: metav1.ObjectMeta{
 				Name: "foo-env",
 				UID:  "2323",
@@ -38,10 +54,10 @@ func TestFunctionServiceCache(t *testing.T) {
 				Builder: fission.Builder{},
 			},
 		},
-		address: "xxx",
+		Address:           "xxx",
-		podName: "yyy",
+		KubernetesObjects: objects,
-		ctime:   now,
+		Ctime:             now,
-		atime:   now,
+		Atime:             now,
 	}
 	err, _ := fsc.Add(*fsvc)
 	if err != nil {
@@ -49,25 +65,25 @@ func TestFunctionServiceCache(t *testing.T) {
 		log.Panicf("Failed to add fsvc: %v", err)
 	}
 
-	f, err := fsc.GetByFunction(fsvc.function)
+	f, err := fsc.GetByFunction(fsvc.Function)
 	if err != nil {
 		fsc.Log()
 		log.Panicf("Failed to get fsvc: %v", err)
 	}
-	fsvc.atime = f.atime
+	fsvc.Atime = f.Atime
-	fsvc.ctime = f.ctime
+	fsvc.Ctime = f.Ctime
-	if *f != *fsvc {
+	if f.Address != fsvc.Address {
 		fsc.Log()
 		log.Panicf("Incorrect fsvc \n(expected: %#v)\n (found: %#v)", fsvc, f)
 	}
 
-	err = fsc.TouchByAddress(fsvc.address)
+	err = fsc.TouchByAddress(fsvc.Address)
 	if err != nil {
 		fsc.Log()
 		log.Panicf("Failed to touch fsvc: %v", err)
 	}
 
-	deleted, err := fsc.DeleteByPod(fsvc.podName, 0)
+	deleted, err := fsc.DeleteOld(fsvc, 0)
 	if err != nil {
 		fsc.Log()
 		log.Panicf("Failed to delete fsvc: %v", err)
@@ -77,7 +93,7 @@ func TestFunctionServiceCache(t *testing.T) {
 		log.Panicf("Did not delete fsvc")
 	}
 
-	_, err = fsc.GetByFunction(fsvc.function)
+	_, err = fsc.GetByFunction(fsvc.Function)
 	if err == nil {
 		fsc.Log()
 		log.Panicf("found fsvc while expecting empty cache: %v", err)

executor/poolmgr/cleanup.go

@@ -26,7 +26,7 @@ import (
 
 // cleanupOldPoolmgrResources looks for resources created by an old
 // poolmgr instance and cleans them up.
-func cleanupOldPoolmgrResources(client *kubernetes.Clientset, namespace string, instanceId string) {
+func CleanupOldPoolmgrResources(client *kubernetes.Clientset, namespace string, instanceId string) {
 	go func() {
 		err := cleanup(client, namespace, instanceId)
 		if err != nil {

executor/poolmgr/gp.go

@@ -36,6 +36,7 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/api"
 	apiv1 "k8s.io/client-go/pkg/api/v1"
 	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
 
@@ -43,6 +44,7 @@ import (
 	"github.com/fission/fission/crd"
 	"github.com/fission/fission/environments/fetcher"
 	fetcherClient "github.com/fission/fission/environments/fetcher/client"
+	"github.com/fission/fission/executor/fscache"
 )
 
 const POOLMGR_INSTANCEID_LABEL string = "poolmgrInstanceId"
@@ -51,14 +53,14 @@ const POD_PHASE_RUNNING string = "Running"
 type (
 	GenericPool struct {
 		env                    *crd.Environment
-		replicas               int32                 // num idle pods
+		replicas               int32                         // num idle pods
-		deployment             *v1beta1.Deployment   // kubernetes deployment
+		deployment             *v1beta1.Deployment           // kubernetes deployment
-		namespace              string                // namespace to keep our resources
+		namespace              string                        // namespace to keep our resources
-		podReadyTimeout        time.Duration         // timeout for generic pods to become ready
+		podReadyTimeout        time.Duration                 // timeout for generic pods to become ready
-		idlePodReapTime        time.Duration         // pods unused for idlePodReapTime are deleted
+		idlePodReapTime        time.Duration                 // pods unused for idlePodReapTime are deleted
-		fsCache                *functionServiceCache // cache funcSvc's by function, address and podname
+		fsCache                *fscache.FunctionServiceCache // cache funcSvc's by function, address and podname
-		useSvc                 bool                  // create k8s service for specialized pods
+		useSvc                 bool                          // create k8s service for specialized pods
-		poolInstanceId         string                // small random string to uniquify pod names
+		poolInstanceId         string                        // small random string to uniquify pod names
 		fetcherImage           string
 		fetcherImagePullPolicy apiv1.PullPolicy
 		runtimeImagePullPolicy apiv1.PullPolicy // pull policy for generic pool to created env deployment
@@ -98,7 +100,7 @@ func MakeGenericPool(
 	env *crd.Environment,
 	initialReplicas int32,
 	namespace string,
-	fsCache *functionServiceCache,
+	fsCache *fscache.FunctionServiceCache,
 	instanceId string) (*GenericPool, error) {
 
 	log.Printf("Creating pool for environment %v", env.Metadata)
@@ -283,6 +285,11 @@ func (gp *GenericPool) scheduleDeletePod(name string) {
 	}()
 }
 
+func IsIPv6(podIP string) bool {
+	ip := net.ParseIP(podIP)
+	return ip != nil && strings.Contains(podIP, ":")
+}
+
 func (gp *GenericPool) getFetcherUrl(podIP string) string {
 	testUrl := os.Getenv("TEST_FETCHER_URL")
 	if len(testUrl) != 0 {
@@ -292,18 +299,35 @@ func (gp *GenericPool) getFetcherUrl(podIP string) string {
 		time.Sleep(5 * time.Second)
 		return testUrl
 	}
-	return fmt.Sprintf("http://%v:8000/", podIP)
+	isv6 := IsIPv6(podIP)
+	var baseUrl string
+	if isv6 == false {
+		baseUrl = fmt.Sprintf("http://%v:8000/", podIP)
+	} else if isv6 == true { // We use bracket if the IP is in IPv6.
+		baseUrl = fmt.Sprintf("http://[%v]:8000/", podIP)
+	}
+	return baseUrl
+
 }
 
 func (gp *GenericPool) getSpecializeUrl(podIP string, version int) string {
 	u := os.Getenv("TEST_SPECIALIZE_URL")
+	isv6 := IsIPv6(podIP)
+	var baseUrl string
 	if len(u) != 0 {
 		return u
 	}
-	if version == 1 {
+	if isv6 == false {
-		return fmt.Sprintf("http://%v:8888/specialize", podIP)
+		baseUrl = fmt.Sprintf("http://%v:8888", podIP)
+	} else if isv6 == true { // We use bracket if the IP is in IPv6.
+		baseUrl = fmt.Sprintf("http://[%v]:8888", podIP)
+	}
+
+	if version == 1 {
+		return fmt.Sprintf("%v/specialize", baseUrl)
+	} else {
+		return fmt.Sprintf("%v/v%v/specialize", baseUrl, version)
 	}
-	return fmt.Sprintf("http://%v:8888/v%v/specialize", podIP, version)
 }
 
 // specializePod chooses a pod, copies the required user-defined function to that pod
@@ -514,7 +538,7 @@ func (gp *GenericPool) createSvc(name string, labels map[string]string) (*apiv1.
 	return svc, err
 }
 
-func (gp *GenericPool) GetFuncSvc(m *metav1.ObjectMeta) (*funcSvc, error) {
+func (gp *GenericPool) GetFuncSvc(m *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
 
 	log.Printf("[%v] Choosing pod from pool", m.Name)
 	newLabels := gp.labelsForFunction(m)
@@ -556,66 +580,69 @@ func (gp *GenericPool) GetFuncSvc(m *metav1.ObjectMeta) (*funcSvc, error) {
 		svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP)
 	}
 
-	fsvc := &funcSvc{
+	kubeObjRefs := []api.ObjectReference{
-		function:    m,
+		{
-		environment: gp.env,
+			Kind:            pod.TypeMeta.Kind,
-		address:     svcHost,
+			Name:            pod.ObjectMeta.Name,
-		podName:     pod.ObjectMeta.Name,
+			APIVersion:      pod.TypeMeta.APIVersion,
-		ctime:       time.Now(),
+			Namespace:       pod.ObjectMeta.Namespace,
-		atime:       time.Now(),
+			ResourceVersion: pod.ObjectMeta.ResourceVersion,
+			UID:             pod.ObjectMeta.UID,
+		},
 	}
 
-	err, existingFsvc := gp.fsCache.Add(*fsvc)
+	fsvc := &fscache.FuncSvc{
+		Function:          m,
+		Environment:       gp.env,
+		Address:           svcHost,
+		KubernetesObjects: kubeObjRefs,
+		Backend:           fscache.POOLMGR,
+		Ctime:             time.Now(),
+		Atime:             time.Now(),
+	}
+
+	err, _ = gp.fsCache.Add(*fsvc)
 	if err != nil {
-		if fe, ok := err.(fission.Error); ok {
-			if fe.Code == fission.ErrorNameExists {
-				// Some other thread beat us to it -- return the other thread's fsvc and clean up
-				// our own.
-				log.Printf("func svc already exists: %v", existingFsvc.podName)
-				go func() {
-					gp.kubernetesClient.CoreV1().Pods(gp.namespace).Delete(fsvc.podName, nil)
-				}()
-				return existingFsvc, nil
-			}
-		}
 		return nil, err
 	}
 	return fsvc, nil
 }
 
-func (gp *GenericPool) CleanupFunctionService(podName string) error {
+func (gp *GenericPool) CleanupFunctionService(obj *fscache.FuncSvc) error {
 	// remove ourselves from fsCache (only if we're still old)
-	deleted, err := gp.fsCache.DeleteByPod(podName, gp.idlePodReapTime)
+	deleted, err := gp.fsCache.DeleteOld(obj, gp.idlePodReapTime)
 	if err != nil {
 		return err
 	}
 
 	if !deleted {
-		log.Printf("Not deleting %v, in use", podName)
+		log.Printf("Not deleting %v, in use", obj.Function)
 		return nil
 	}
 
-	pod, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).Get(podName, metav1.GetOptions{})
+	for _, kubeobj := range obj.KubernetesObjects {
-	if err != nil {
+		pod, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).Get(kubeobj.Name, metav1.GetOptions{})
-		return err
-	}
 
-	loggerUrl := fmt.Sprintf("http://%s:1234/v1/log/%s", pod.Spec.NodeName, pod.Name)
+		loggerUrl := fmt.Sprintf("http://%s:1234/v1/log/%s", pod.Spec.NodeName, pod.Name)
-	req, err := http.NewRequest("DELETE", loggerUrl, nil)
+		req, err := http.NewRequest("DELETE", loggerUrl, nil)
-	resp, err := http.DefaultClient.Do(req)
+		resp, err := http.DefaultClient.Do(req)
-	if err != nil {
+		if err != nil {
-		log.Printf("Error from %s daemonset logger: %v", pod.Spec.NodeName, err)
+			log.Printf("Error from %s daemonset logger: %v", pod.Spec.NodeName, err)
-	} else {
+		} else {
-		if resp.StatusCode != 200 {
+			if resp.StatusCode != 200 {
-			log.Printf("Received not http 200(OK) status from %s daemonset logger: %s", pod.Spec.NodeName, resp.Status)
+				log.Printf("Received not http 200(OK) status from %s daemonset logger: %s", pod.Spec.NodeName, resp.Status)
+			}
+			resp.Body.Close()
 		}
-		resp.Body.Close()
-	}
 
-	// delete pod
+		// delete pod
-	err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Delete(podName, nil)
+		err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Delete(kubeobj.Name, nil)
-	if err != nil {
+		if err != nil {
-		return err
+			return err
+		}
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -624,16 +651,19 @@ func (gp *GenericPool) CleanupFunctionService(podName string) error {
 func (gp *GenericPool) idlePodReaper() {
 	for {
 		time.Sleep(time.Minute)
-		podNames, err := gp.fsCache.ListOld(&gp.env.Metadata, gp.idlePodReapTime)
+		funcSvcs, err := gp.fsCache.ListOld(&gp.env.Metadata, gp.idlePodReapTime)
 		if err != nil {
 			log.Printf("Error reaping idle pods: %v", err)
 			continue
 		}
-		for _, podName := range podNames {
+		for _, obj := range funcSvcs {
-			log.Printf("Reaping idle pod '%v'", podName)
+			err := gp.CleanupFunctionService(obj)
-			err := gp.CleanupFunctionService(podName)
 			if err != nil {
-				log.Printf("Error deleting idle pod '%v': %v", podName, err)
+				log.Printf("Error deleting Kubernetes objects for fsvc '%v': %v", obj, err)
+				log.Printf("Object Name| Object Kind | Object Space")
+				for _, kubeobj := range obj.KubernetesObjects {
+					log.Printf("%v | %v | %v", kubeobj.Name, kubeobj.Kind, kubeobj.Namespace)
+				}
 			}
 		}
 	}

executor/poolmgr/gpm.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/crd"
+	"github.com/fission/fission/executor/fscache"
 )
 
 type requestType int
@@ -39,10 +40,11 @@ type (
 		pools            map[string]*GenericPool
 		kubernetesClient *kubernetes.Clientset
 		namespace        string
-		fissionClient    *crd.FissionClient
+
-		fsCache          *functionServiceCache
+		fissionClient  *crd.FissionClient
-		instanceId       string
+		fsCache        *fscache.FunctionServiceCache
-		requestChannel   chan *request
+		instanceId     string
+		requestChannel chan *request
 	}
 	request struct {
 		requestType
@@ -61,7 +63,7 @@ func MakeGenericPoolManager(
 	kubernetesClient *kubernetes.Clientset,
 	fissionNamespace string,
 	functionNamespace string,
-	fsCache *functionServiceCache,
+	fsCache *fscache.FunctionServiceCache,
 	instanceId string) *GenericPoolManager {
 
 	gpm := &GenericPoolManager{

fission-bundle/main.go

@@ -6,11 +6,12 @@ import (
 	"strconv"
 
 	"github.com/docopt/docopt-go"
+
 	"github.com/fission/fission/buildermgr"
 	"github.com/fission/fission/controller"
+	"github.com/fission/fission/executor"
 	"github.com/fission/fission/kubewatcher"
 	"github.com/fission/fission/mqtrigger"
-	"github.com/fission/fission/poolmgr"
 	"github.com/fission/fission/router"
 	"github.com/fission/fission/storagesvc"
 	"github.com/fission/fission/timer"
@@ -21,15 +22,15 @@ func runController(port int) {
 	log.Fatalf("Error: Controller exited.")
 }
 
-func runRouter(port int, poolmgrUrl string) {
+func runRouter(port int, executorUrl string) {
-	router.Start(port, poolmgrUrl)
+	router.Start(port, executorUrl)
 	log.Fatalf("Error: Router exited.")
 }
 
-func runPoolmgr(port int, fissionNamespace, functionNamespace string) {
+func runExecutor(port int, fissionNamespace, functionNamespace string) {
-	err := poolmgr.StartPoolmgr(fissionNamespace, functionNamespace, port)
+	err := executor.StartExecutor(fissionNamespace, functionNamespace, port)
 	if err != nil {
-		log.Fatalf("Error starting poolmgr: %v", err)
+		log.Fatalf("Error starting executor: %v", err)
 	}
 }
 
@@ -88,18 +89,18 @@ func getStringArgWithDefault(arg interface{}, defaultValue string) string {
 }
 
 func main() {
-	usage := `fission-bundle: Package of all fission microservices: controller, router, poolmgr.
+	usage := `fission-bundle: Package of all fission microservices: controller, router, executor.
 
 Use it to start one or more of the fission servers:
 
  Controller is a stateless API frontend for fission resources.
 
  Pool manager maintains a pool of generalized function containers, and
- specializes them on-demand. Poolmgr must be run from a pod in a
+ specializes them on-demand. Executor must be run from a pod in a
  Kubernetes cluster.
 
  Router implements HTTP triggers: it routes to running instances,
- working with the controller and poolmgr.
+ working with the controller and executor.
 
  Kubewatcher implements Kubernetes Watch triggers: it watches
  Kubernetes resources and invokes functions described in the
@@ -111,8 +112,8 @@ Use it to start one or more of the fission servers:
 
 Usage:
   fission-bundle --controllerPort=<port>
-  fission-bundle --routerPort=<port> [--poolmgrUrl=<url>]
+  fission-bundle --routerPort=<port> [--executorUrl=<url>]
-  fission-bundle --poolmgrPort=<port> [--namespace=<namespace>] [--fission-namespace=<namespace>]
+  fission-bundle --executorPort=<port> [--namespace=<namespace>] [--fission-namespace=<namespace>]
   fission-bundle --kubewatcher [--routerUrl=<url>]
   fission-bundle --storageServicePort=<port> --filePath=<filePath>
   fission-bundle --builderMgrPort=<port> [--storageSvcUrl=<url>] [--envbuilder-namespace=<namespace>]
@@ -121,10 +122,10 @@ Usage:
 Options:
   --controllerPort=<port>         Port that the controller should listen on.
   --routerPort=<port>             Port that the router should listen on.
-  --poolmgrPort=<port>            Port that the poolmgr should listen on.
+  --executorPort=<port>           Port that the executor should listen on.
   --storageServicePort=<port>     Port that the storage service should listen on.
   --builderMgrPort=<port>         Port that the buildermgr should listen on.
-  --poolmgrUrl=<url>              Poolmgr URL. Not required if --poolmgrPort is specified.
+  --executorUrl=<url>             Executor URL. Not required if --executorPort is specified.
   --routerUrl=<url>               Router URL.
   --etcdUrl=<etcdUrl>             Etcd URL.
   --storageSvcUrl=<url>           StorageService URL.
@@ -143,7 +144,7 @@ Options:
 	fissionNs := getStringArgWithDefault(arguments["--fission-namespace"], "fission")
 	envBuilderNs := getStringArgWithDefault(arguments["--envbuilder-namespace"], "fission-builder")
 
-	poolmgrUrl := getStringArgWithDefault(arguments["--poolmgrUrl"], "http://poolmgr.fission")
+	executorUrl := getStringArgWithDefault(arguments["--executorUrl"], "http://executor.fission")
 	routerUrl := getStringArgWithDefault(arguments["--routerUrl"], "http://router.fission")
 	storageSvcUrl := getStringArgWithDefault(arguments["--storageSvcUrl"], "http://storagesvc.fission")
 
@@ -154,12 +155,12 @@ Options:
 
 	if arguments["--routerPort"] != nil {
 		port := getPort(arguments["--routerPort"])
-		runRouter(port, poolmgrUrl)
+		runRouter(port, executorUrl)
 	}
 
-	if arguments["--poolmgrPort"] != nil {
+	if arguments["--executorPort"] != nil {
-		port := getPort(arguments["--poolmgrPort"])
+		port := getPort(arguments["--executorPort"])
-		runPoolmgr(port, fissionNs, functionNs)
+		runExecutor(port, fissionNs, functionNs)
 	}
 
 	if arguments["--kubewatcher"] == true {

fission/common.go

@@ -17,13 +17,24 @@ limitations under the License.
 package main
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 
+	uuid "github.com/satori/go.uuid"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission"
 	"github.com/fission/fission/controller/client"
+	"github.com/fission/fission/crd"
+	storageSvcClient "github.com/fission/fission/storagesvc/client"
 )
 
 func fatal(msg string) {
@@ -86,3 +97,169 @@ func httpRequest(method, url, body string, headers []string) *http.Response {
 
 	return resp
 }
+
+func fileSize(filePath string) int64 {
+	info, err := os.Stat(filePath)
+	checkErr(err, fmt.Sprintf("stat %v", filePath))
+	return info.Size()
+}
+
+// upload a file and return a fission.Archive
+func createArchive(client *client.Client, fileName string) *fission.Archive {
+	var archive fission.Archive
+
+	// fetch archive from arbitrary url if fileName is a url
+	if strings.HasPrefix(fileName, "http://") || strings.HasPrefix(fileName, "https://") {
+		fileName = downloadToTempFile(fileName)
+	}
+
+	if fileSize(fileName) < fission.ArchiveLiteralSizeLimit {
+		contents := getContents(fileName)
+		archive.Type = fission.ArchiveTypeLiteral
+		archive.Literal = contents
+	} else {
+		u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage"
+		ssClient := storageSvcClient.MakeClient(u)
+
+		// TODO add a progress bar
+		id, err := ssClient.Upload(fileName, nil)
+		checkErr(err, fmt.Sprintf("upload file %v", fileName))
+
+		archiveUrl := ssClient.GetUrl(id)
+
+		archive.Type = fission.ArchiveTypeUrl
+		archive.URL = archiveUrl
+
+		f, err := os.Open(fileName)
+		if err != nil {
+			checkErr(err, fmt.Sprintf("find file %v", fileName))
+		}
+		defer f.Close()
+
+		h := sha256.New()
+		if _, err := io.Copy(h, f); err != nil {
+			checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
+		}
+
+		archive.Checksum = fission.Checksum{
+			Type: fission.ChecksumTypeSHA256,
+			Sum:  hex.EncodeToString(h.Sum(nil)),
+		}
+	}
+	return &archive
+}
+
+func createPackage(client *client.Client, envName, srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta {
+	pkgSpec := fission.PackageSpec{
+		Environment: fission.EnvironmentReference{
+			Namespace: metav1.NamespaceDefault,
+			Name:      envName,
+		},
+	}
+	var pkgStatus fission.BuildStatus = fission.BuildStatusSucceeded
+
+	if len(deployArchiveName) > 0 {
+		pkgSpec.Deployment = *createArchive(client, deployArchiveName)
+	}
+	if len(srcArchiveName) > 0 {
+		pkgSpec.Source = *createArchive(client, srcArchiveName)
+		// set pending status to package
+		pkgStatus = fission.BuildStatusPending
+	}
+
+	if len(buildcmd) > 0 {
+		pkgSpec.BuildCommand = buildcmd
+	}
+
+	pkgName := strings.ToLower(uuid.NewV4().String())
+	pkg := &crd.Package{
+		Metadata: metav1.ObjectMeta{
+			Name:      pkgName,
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: pkgSpec,
+		Status: fission.PackageStatus{
+			BuildStatus: pkgStatus,
+		},
+	}
+	pkgMetadata, err := client.PackageCreate(pkg)
+	checkErr(err, "create package")
+	return pkgMetadata
+}
+
+func getContents(filePath string) []byte {
+	var code []byte
+	var err error
+
+	code, err = ioutil.ReadFile(filePath)
+	checkErr(err, fmt.Sprintf("read %v", filePath))
+	return code
+}
+
+func getTempDir() (string, error) {
+	tmpDir := uuid.NewV4().String()
+	tmpPath := filepath.Join(os.TempDir(), tmpDir)
+	err := os.Mkdir(tmpPath, 0744)
+	return tmpPath, err
+}
+
+func writeArchiveToFile(fileName string, reader io.Reader) error {
+	tmpDir, err := getTempDir()
+	if err != nil {
+		return err
+	}
+
+	path := filepath.Join(tmpDir, fileName+".tmp")
+	w, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(w, reader)
+	if err != nil {
+		return err
+	}
+	err = os.Chmod(path, 0644)
+	if err != nil {
+		return err
+	}
+
+	err = os.Rename(path, fileName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// downloadToTempFile fetches archive file from arbitrary url
+// and write it to temp file for further usage
+func downloadToTempFile(fileUrl string) string {
+	reader, err := downloadURL(fileUrl)
+	defer reader.Close()
+	checkErr(err, fmt.Sprintf("download from url: %v", fileUrl))
+
+	tmpDir, err := getTempDir()
+	checkErr(err, "create temp directory")
+
+	tmpFilename := uuid.NewV4().String()
+	destination := filepath.Join(tmpDir, tmpFilename)
+	err = os.Mkdir(tmpDir, 0744)
+	checkErr(err, "create temp directory")
+
+	err = writeArchiveToFile(destination, reader)
+	checkErr(err, "write archive to file")
+
+	return destination
+}
+
+// downloadURL downloads file from given url
+func downloadURL(fileUrl string) (io.ReadCloser, error) {
+	resp, err := http.Get(fileUrl)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%v - HTTP response returned non 200 status", resp.StatusCode)
+	}
+	return resp.Body, nil
+}

fission/function.go

@@ -18,139 +18,24 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
-	"strings"
 	"text/tabwriter"
 	"time"
 
-	"github.com/dchest/uniuri"
 	"github.com/satori/go.uuid"
 	"github.com/urfave/cli"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/fission/fission"
-	"github.com/fission/fission/controller/client"
 	"github.com/fission/fission/crd"
 	"github.com/fission/fission/fission/logdb"
-	storageSvcClient "github.com/fission/fission/storagesvc/client"
 )
 
-func fileSize(filePath string) int64 {
-	info, err := os.Stat(filePath)
-	checkErr(err, fmt.Sprintf("stat %v", filePath))
-	return info.Size()
-}
-
-// upload a file and return a fission.Archive
-func createArchive(client *client.Client, fileName string) *fission.Archive {
-	var archive fission.Archive
-	if fileSize(fileName) < fission.ArchiveLiteralSizeLimit {
-		contents := getContents(fileName)
-		archive.Type = fission.ArchiveTypeLiteral
-		archive.Literal = contents
-	} else {
-		u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage"
-		ssClient := storageSvcClient.MakeClient(u)
-
-		// TODO add a progress bar
-		id, err := ssClient.Upload(fileName, nil)
-		checkErr(err, fmt.Sprintf("upload file %v", fileName))
-
-		archiveUrl := ssClient.GetUrl(id)
-
-		archive.Type = fission.ArchiveTypeUrl
-		archive.URL = archiveUrl
-
-		f, err := os.Open(fileName)
-		if err != nil {
-			checkErr(err, fmt.Sprintf("find file %v", fileName))
-		}
-		defer f.Close()
-
-		h := sha256.New()
-		if _, err := io.Copy(h, f); err != nil {
-			checkErr(err, fmt.Sprintf("calculate checksum for file %v", fileName))
-		}
-
-		archive.Checksum = fission.Checksum{
-			Type: fission.ChecksumTypeSHA256,
-			Sum:  hex.EncodeToString(h.Sum(nil)),
-		}
-	}
-	return &archive
-}
-
-func createPackage(client *client.Client, fnName, envName, srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta {
-	pkgSpec := fission.PackageSpec{
-		Environment: fission.EnvironmentReference{
-			Namespace: metav1.NamespaceDefault,
-			Name:      envName,
-		},
-	}
-
-	var pkgStatus fission.BuildStatus
-
-	if len(deployArchiveName) > 0 {
-		pkgSpec.Deployment = *createArchive(client, deployArchiveName)
-	}
-	if len(srcArchiveName) > 0 {
-		pkgSpec.Source = *createArchive(client, srcArchiveName)
-	}
-
-	// start a build only when a package has no deploy archive
-	if len(srcArchiveName) > 0 && len(deployArchiveName) == 0 {
-		pkgStatus = fission.BuildStatusPending
-	} else {
-		pkgStatus = fission.BuildStatusNone
-	}
-
-	if len(buildcmd) > 0 {
-		pkgSpec.BuildCommand = buildcmd
-	}
-
-	fnList, err := json.Marshal([]string{fnName})
-	checkErr(err, "encode json")
-
-	annotation := map[string]string{
-		"createdForFunction": fnName,
-		"usedByFunctions":    string(fnList),
-	}
-
-	pkgName := strings.ToLower(fmt.Sprintf("%v-%v", fnName, uniuri.NewLen(6)))
-	pkg := &crd.Package{
-		Metadata: metav1.ObjectMeta{
-			Name:        pkgName,
-			Namespace:   metav1.NamespaceDefault,
-			Annotations: annotation,
-		},
-		Spec: pkgSpec,
-		Status: fission.PackageStatus{
-			BuildStatus: pkgStatus,
-		},
-	}
-	pkgMetadata, err := client.PackageCreate(pkg)
-	checkErr(err, "create package")
-	return pkgMetadata
-}
-
-func getContents(filePath string) []byte {
-	var code []byte
-	var err error
-
-	code, err = ioutil.ReadFile(filePath)
-	checkErr(err, fmt.Sprintf("read %v", filePath))
-	return code
-}
-
 func printPodLogs(c *cli.Context) error {
 	fnName := c.String("name")
 	if len(fnName) == 0 {
@@ -191,25 +76,51 @@ func fnCreate(c *cli.Context) error {
 		fatal("Need --name argument.")
 	}
 
-	envName := c.String("env")
+	fnList, err := client.FunctionList()
-	if len(envName) == 0 {
+	checkErr(err, "get function list")
-		fatal("Need --env argument.")
+	// check function existence before creating package
+	for _, fn := range fnList {
+		if fn.Metadata.Name == fnName {
+			fatal("A function with the same name already exists.")
+		}
 	}
-
-	srcArchiveName := c.String("src")
-	deployArchiveName := c.String("code")
-	if len(deployArchiveName) == 0 {
-		deployArchiveName = c.String("deploy")
-	}
-
-	if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 {
-		fatal("Need --code or --deploy to specify deployment archive, or use --src to specify source archive.")
-	}
-
 	entrypoint := c.String("entrypoint")
-	buildcmd := c.String("buildcmd")
+	pkgName := c.String("pkg")
 
-	pkgMetadata := createPackage(client, fnName, envName, srcArchiveName, deployArchiveName, buildcmd)
+	var pkgMetadata *metav1.ObjectMeta
+	var envName string
+
+	if len(pkgName) > 0 {
+		// use existing package
+		pkg, err := client.PackageGet(&metav1.ObjectMeta{
+			Namespace: metav1.NamespaceDefault,
+			Name:      pkgName,
+		})
+		checkErr(err, fmt.Sprintf("read package '%v'", pkgName))
+		pkgMetadata = &pkg.Metadata
+		envName = pkg.Spec.Environment.Name
+	} else {
+		// need to specify environment for creating new package
+		envName = c.String("env")
+		if len(envName) == 0 {
+			fatal("Need --env argument.")
+		}
+
+		srcArchiveName := c.String("src")
+		deployArchiveName := c.String("code")
+		if len(deployArchiveName) == 0 {
+			deployArchiveName = c.String("deploy")
+		}
+		// fatal when both src & deploy archive are empty
+		if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 {
+			fatal("Need --deploy or --src argument.")
+		}
+
+		buildcmd := c.String("buildcmd")
+
+		// create new package
+		pkgMetadata = createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
+	}
 
 	function := &crd.Function{
 		Metadata: metav1.ObjectMeta{
@@ -232,7 +143,7 @@ func fnCreate(c *cli.Context) error {
 		},
 	}
 
-	_, err := client.FunctionCreate(function)
+	_, err = client.FunctionCreate(function)
 	checkErr(err, "create function")
 
 	fmt.Printf("function '%v' created\n", fnName)
@@ -319,15 +230,19 @@ func fnGetMeta(c *cli.Context) error {
 func fnUpdate(c *cli.Context) error {
 	client := getClient(c.GlobalString("server"))
 
+	if len(c.String("package")) > 0 {
+		fatal("--package is deprecated, please use --deploy instead.")
+	}
+
+	if len(c.String("srcpkg")) > 0 {
+		fatal("--srcpkg is deprecated, please use --src instead.")
+	}
+
 	fnName := c.String("name")
 	if len(fnName) == 0 {
 		fatal("Need name of function, use --name")
 	}
 
-	if len(c.String("package")) > 0 {
-		fatal("--package is deprecated, please use --deploy instead.")
-	}
-
 	function, err := client.FunctionGet(&metav1.ObjectMeta{
 		Name:      fnName,
 		Namespace: metav1.NamespaceDefault,
@@ -340,43 +255,65 @@ func fnUpdate(c *cli.Context) error {
 		deployArchiveName = c.String("deploy")
 	}
 	srcArchiveName := c.String("src")
+	pkgName := c.String("pkg")
+	entrypoint := c.String("entrypoint")
+	buildcmd := c.String("buildcmd")
+	force := c.Bool("force")
 
-	if len(envName) == 0 && len(deployArchiveName) == 0 && len(srcArchiveName) == 0 {
+	if len(envName) == 0 && len(deployArchiveName) == 0 && len(srcArchiveName) == 0 && len(pkgName) == 0 &&
-		fatal("Need --env or --code or --package or --deploy argument.")
+		len(entrypoint) == 0 && len(buildcmd) == 0 {
+		fatal("Need --env or --deploy or --src or --pkg or --entrypoint or --buildcmd argument.")
 	}
 
 	if len(envName) > 0 {
 		function.Spec.Environment.Name = envName
 	}
 
-	entrypoint := c.String("entrypoint")
 	if len(entrypoint) > 0 {
 		function.Spec.Package.FunctionName = entrypoint
 	}
 
-	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+	if len(pkgName) == 0 {
-		Name:      function.Spec.Package.PackageRef.Name,
+		pkgName = function.Spec.Package.PackageRef.Name
-		Namespace: function.Spec.Package.PackageRef.Namespace,
-	})
-	checkErr(err, fmt.Sprintf("read package '%v'", function.Spec.Package.PackageRef.Name))
-
-	buildcmd := c.String("buildcmd")
-	if len(buildcmd) == 0 {
-		// use previous build command if not specified.
-		buildcmd = pkg.Spec.BuildCommand
 	}
 
-	if len(deployArchiveName) > 0 || len(srcArchiveName) > 0 {
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
-		// create a new package for function
+		Namespace: metav1.NamespaceDefault,
-		pkgMetadata := createPackage(client, function.Metadata.Name,
+		Name:      pkgName,
-			function.Spec.Environment.Name, srcArchiveName, deployArchiveName, buildcmd)
+	})
+	checkErr(err, fmt.Sprintf("read package '%v'", pkgName))
 
-		// update function spec with resource version
+	pkgMetadata := &pkg.Metadata
-		function.Spec.Package.PackageRef = fission.PackageRef{
+
-			Namespace:       pkgMetadata.Namespace,
+	if len(deployArchiveName) != 0 || len(srcArchiveName) != 0 || len(buildcmd) != 0 || len(envName) != 0 {
-			Name:            pkgMetadata.Name,
+		fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name)
-			ResourceVersion: pkgMetadata.ResourceVersion,
+		checkErr(err, "get function list")
+
+		if !force && len(fnList) > 1 {
+			fatal("Package is used by multiple functions, use --force to force update")
 		}
+
+		pkgMetadata = updatePackage(client, pkg, envName, srcArchiveName, deployArchiveName, buildcmd)
+		checkErr(err, fmt.Sprintf("update package '%v'", pkgName))
+
+		fmt.Printf("package '%v' updated\n", pkgMetadata.GetName())
+
+		// update resource version of package reference of functions that shared the same package
+		for _, fn := range fnList {
+			// ignore the update for current function here, it will be updated later.
+			if fn.Metadata.Name != fnName {
+				fn.Spec.Package.PackageRef.ResourceVersion = pkgMetadata.ResourceVersion
+				_, err := client.FunctionUpdate(&fn)
+				checkErr(err, "update function")
+			}
+		}
+	}
+
+	// update function spec with new package metadata
+	function.Spec.Package.PackageRef = fission.PackageRef{
+		Namespace:       pkgMetadata.Namespace,
+		Name:            pkgMetadata.Name,
+		ResourceVersion: pkgMetadata.ResourceVersion,
 	}
 
 	_, err = client.FunctionUpdate(function)
@@ -444,6 +381,11 @@ func fnLogs(c *cli.Context) error {
 		Namespace: metav1.NamespaceDefault,
 	}
 
+	recordLimit := c.Int("recordcount")
+	if recordLimit <= 0 {
+		recordLimit = 1000
+	}
+
 	f, err := client.FunctionGet(m)
 	checkErr(err, "get function")
 
@@ -463,10 +405,11 @@ func fnLogs(c *cli.Context) error {
 			select {
 			case <-requestChan:
 				logFilter := logdb.LogFilter{
-					Pod:      fnPod,
+					Pod:         fnPod,
-					Function: f.Metadata.Name,
+					Function:    f.Metadata.Name,
-					FuncUid:  string(f.Metadata.UID),
+					FuncUid:     string(f.Metadata.UID),
-					Since:    t,
+					Since:       t,
+					RecordLimit: recordLimit,
 				}
 				logEntries, err := logDB.GetLogs(logFilter)
 				if err != nil {

fission/httptrigger.go

@@ -157,10 +157,10 @@ func htList(c *cli.Context) error {
 
 	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
 
-	fmt.Fprintf(w, "%v\t%v\t%v\t%v\n", "NAME", "METHOD", "URL", "FUNCTION_NAME")
+	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n", "NAME", "METHOD", "HOST", "URL", "FUNCTION_NAME")
 	for _, ht := range hts {
-		fmt.Fprintf(w, "%v\t%v\t%v\t%v\n",
+		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\n",
-			ht.Metadata.Name, ht.Spec.Method, ht.Spec.RelativeURL, ht.Spec.FunctionReference.Name)
+			ht.Metadata.Name, ht.Spec.Method, ht.Spec.Host, ht.Spec.RelativeURL, ht.Spec.FunctionReference.Name)
 	}
 	w.Flush()
 

fission/logdb/influxdb.go

@@ -22,6 +22,8 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -65,6 +67,15 @@ func (influx InfluxDB) GetPods(filter LogFilter) ([]string, error) {
 	return pods, nil
 }
 
+func makeIndexMap(cols []string) map[string]int {
+	indexMap := make(map[string]int, len(cols))
+	for i := range cols {
+		indexMap[cols[i]] = i
+	}
+
+	return indexMap
+}
+
 func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
 	timestamp := filter.Since.UnixNano()
 	var queryCmd string
@@ -74,12 +85,13 @@ func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
 	parameters := make(map[string]interface{})
 	parameters["funcuid"] = filter.FuncUid
 	parameters["time"] = timestamp
+	//the parameters above are only for the where clause and do not work with LIMIT
 
 	if filter.Pod != "" {
-		queryCmd = "select * from \"log\" where \"funcuid\" = $funcuid AND \"pod\" = $pod AND \"time\" > $time ORDER BY time ASC"
+		queryCmd = "select * from \"log\" where \"funcuid\" = $funcuid AND \"pod\" = $pod AND \"time\" > $time LIMIT " + strconv.Itoa(filter.RecordLimit)
 		parameters["pod"] = filter.Pod
 	} else {
-		queryCmd = "select * from \"log\" where \"funcuid\" = $funcuid AND \"time\" > $time ORDER BY time ASC"
+		queryCmd = "select * from \"log\" where \"funcuid\" = $funcuid AND \"time\" > $time LIMIT " + strconv.Itoa(filter.RecordLimit)
 	}
 
 	query := influxdbClient.NewQueryWithParameters(queryCmd, INFLUXDB_DATABASE, "", parameters)
@@ -90,25 +102,53 @@ func (influx InfluxDB) GetLogs(filter LogFilter) ([]LogEntry, error) {
 	}
 	for _, r := range response.Results {
 		for _, series := range r.Series {
+
+			//create map of columns to row indeces
+			indexMap := makeIndexMap(series.Columns)
+
+			container := indexMap["docker_container_id"]
+			functionName := indexMap["kubernetes_labels_functionName"]
+			funcuid := indexMap["kubernetes_labels_functionUid"]
+			logMessage := indexMap["log"]
+			nameSpace := indexMap["kubernetes_namespace_name"]
+			podName := indexMap["kubernetes_pod_name"]
+			stream := indexMap["stream"]
+			seq := indexMap["_seq"]
+
 			for _, row := range series.Values {
 				t, err := time.Parse(time.RFC3339, row[0].(string))
 				if err != nil {
 					log.Fatal(err)
 				}
+				seqNum, err := strconv.Atoi(row[seq].(string))
+				if err != nil {
+					return logEntries, err
+				}
 				logEntries = append(logEntries, LogEntry{
 					//The attributes of the LogEntry are selected as relative to their position in InfluxDB's line protocol response
 					Timestamp: t,
-					Container: row[2].(string),                            //docker_container_id
+					Container: row[container].(string),                            //docker_container_id
-					FuncName:  row[8].(string),                            //kubernetes_labels_functionName
+					FuncName:  row[functionName].(string),                         //kubernetes_labels_functionName
-					FuncUid:   row[3].(string),                            //funcuid
+					FuncUid:   row[funcuid].(string),                              //funcuid
-					Message:   strings.TrimSuffix(row[17].(string), "\n"), //log field
+					Message:   strings.TrimSuffix(row[logMessage].(string), "\n"), //log field
-					Namespace: row[14].(string),                           //kubernetes_namespace_name
+					Namespace: row[nameSpace].(string),                            //kubernetes_namespace_name
-					Pod:       row[15].(string),                           //kubernetes_pod_name
+					Pod:       row[podName].(string),                              //kubernetes_pod_name
-					Stream:    row[18].(string),                           //stream
+					Stream:    row[stream].(string),                               //stream
+					Sequence:  seqNum,                                             //sequence tag
 				})
 			}
 		}
 	}
+	sort.Slice(logEntries, func(i, j int) bool {
+
+		if logEntries[i].Timestamp.Before(logEntries[j].Timestamp) {
+			return true
+		}
+		if logEntries[j].Timestamp.Before(logEntries[i].Timestamp) {
+			return false
+		}
+		return logEntries[i].Sequence < logEntries[j].Sequence
+	})
 	return logEntries, nil
 }
 

fission/logdb/logdb.go

@@ -32,16 +32,18 @@ type LogDatabase interface {
 }
 
 type LogFilter struct {
-	Pod      string
+	Pod         string
-	Function string
+	Function    string
-	FuncUid  string
+	FuncUid     string
-	Since    time.Time
+	Since       time.Time
+	RecordLimit int
 }
 
 type LogEntry struct {
 	Timestamp time.Time
 	Message   string
 	Stream    string
+	Sequence  int
 	Container string
 	Namespace string
 	FuncName  string

fission/main.go

@@ -26,7 +26,7 @@ func main() {
 	app := cli.NewApp()
 	app.Name = "fission"
 	app.Usage = "Serverless functions for Kubernetes"
-	app.Version = "0.4.0"
+	app.Version = "0.4.1"
 
 	app.Flags = []cli.Flag{
 		cli.StringFlag{Name: "server", Usage: "Fission server URL", EnvVar: "FISSION_URL"},
@@ -43,6 +43,7 @@ func main() {
 	fnPackageFlag := cli.StringFlag{Name: "package", Usage: "(Deprecated) local path or URL for binary package"}
 	fnDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "local path or URL for deployment archive"}
 	fnSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "local path or URL for source archive"}
+	fnPkgNameFlag := cli.StringFlag{Name: "pkgname, pkg", Usage: "Name of the existing package (--deploy and --src and --env will be ignored)"}
 	fnPodFlag := cli.StringFlag{Name: "pod", Usage: "function pod name, optional (use latest if unspecified)"}
 	fnFollowFlag := cli.BoolFlag{Name: "follow, f", Usage: "specify if the logs should be streamed"}
 	fnDetailFlag := cli.BoolFlag{Name: "detail, d", Usage: "display detailed information"}
@@ -51,15 +52,17 @@ func main() {
 	fnHeaderFlag := cli.StringSliceFlag{Name: "header, H", Usage: "request headers"}
 	fnEntryPointFlag := cli.StringFlag{Name: "entrypoint", Usage: "entry point for environment v2 to load with"}
 	fnBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "build command for builder to run with"}
+	fnLogCountFlag := cli.StringFlag{Name: "recordcount", Usage: "the n most recent log records"}
+	fnForceFlag := cli.BoolFlag{Name: "force", Usage: "Force update a package even if it is used by one or more functions"}
 
 	fnSubcommands := []cli.Command{
-		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
+		{Name: "create", Usage: "Create new function (and optionally, an HTTP route to it)", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag, fnPkgNameFlag, htUrlFlag, htMethodFlag}, Action: fnCreate},
 		{Name: "get", Usage: "Get function source code", Flags: []cli.Flag{fnNameFlag}, Action: fnGet},
 		{Name: "getmeta", Usage: "Get function metadata", Flags: []cli.Flag{fnNameFlag}, Action: fnGetMeta},
-		{Name: "update", Usage: "Update function source code", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnBuildCmdFlag}, Action: fnUpdate},
+		{Name: "update", Usage: "Update function", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, fnDeployArchiveFlag, fnEntryPointFlag, fnPkgNameFlag, fnBuildCmdFlag, fnForceFlag}, Action: fnUpdate},
 		{Name: "delete", Usage: "Delete function", Flags: []cli.Flag{fnNameFlag}, Action: fnDelete},
 		{Name: "list", Usage: "List all functions", Flags: []cli.Flag{}, Action: fnList},
-		{Name: "logs", Usage: "Display function logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag}, Action: fnLogs},
+		{Name: "logs", Usage: "Display function logs", Flags: []cli.Flag{fnNameFlag, fnPodFlag, fnFollowFlag, fnDetailFlag, fnLogDBTypeFlag, fnLogCountFlag}, Action: fnLogs},
 		{Name: "pods", Usage: "Display function pods", Flags: []cli.Flag{fnNameFlag, fnLogDBTypeFlag}, Action: fnPods},
 		{Name: "test", Usage: "Test a function", Flags: []cli.Flag{fnNameFlag, fnEnvNameFlag, fnCodeFlag, fnPackageFlag, fnSrcArchiveFlag, htMethodFlag, fnBodyFlag, fnHeaderFlag}, Action: fnTest},
 	}
@@ -130,6 +133,24 @@ func main() {
 		{Name: "list", Usage: "List all watches", Flags: []cli.Flag{}, Action: wList},
 	}
 
+	// packages
+	pkgNameFlag := cli.StringFlag{Name: "name", Usage: "Package name"}
+	pkgForceFlag := cli.BoolFlag{Name: "force, f", Usage: "Force update a package even if it is used by one or more functions"}
+	pkgEnvironmentFlag := cli.StringFlag{Name: "env", Usage: "Environment name"}
+	pkgSrcArchiveFlag := cli.StringFlag{Name: "sourcearchive, src", Usage: "Local path or URL for source archive"}
+	pkgDeployArchiveFlag := cli.StringFlag{Name: "deployarchive, deploy", Usage: "Local path or URL for binary archive"}
+	pkgBuildCmdFlag := cli.StringFlag{Name: "buildcmd", Usage: "Build command for builder to run with"}
+	pkgOutputFlag := cli.StringFlag{Name: "output, o", Usage: "Output filename to save archive content"}
+	pkgSubCommands := []cli.Command{
+		{Name: "create", Usage: "Create new package", Flags: []cli.Flag{pkgEnvironmentFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag}, Action: pkgCreate},
+		{Name: "update", Usage: "Update package", Flags: []cli.Flag{pkgNameFlag, pkgEnvironmentFlag, pkgSrcArchiveFlag, pkgDeployArchiveFlag, pkgBuildCmdFlag, pkgForceFlag}, Action: pkgUpdate},
+		{Name: "getsrc", Usage: "Get source archive content", Flags: []cli.Flag{pkgNameFlag, pkgOutputFlag}, Action: pkgSourceGet},
+		{Name: "getdeploy", Usage: "Get deployment archive content", Flags: []cli.Flag{pkgNameFlag, pkgOutputFlag}, Action: pkgDeployGet},
+		{Name: "info", Usage: "Show package information", Flags: []cli.Flag{pkgNameFlag}, Action: pkgInfo},
+		{Name: "list", Usage: "List all packages", Flags: []cli.Flag{}, Action: pkgList},
+		{Name: "delete", Usage: "Delete package", Flags: []cli.Flag{pkgNameFlag, pkgForceFlag}, Action: pkgDelete},
+	}
+
 	upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"}
 	upgradeSubCommands := []cli.Command{
 		{Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState},
@@ -150,6 +171,7 @@ func main() {
 		{Name: "mqtrigger", Aliases: []string{"mqt", "messagequeue"}, Usage: "Manage message queue triggers for functions", Subcommands: mqtSubcommands},
 		{Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands},
 		{Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands},
+		{Name: "package", Aliases: []string{"pkg"}, Usage: "Manage packages", Subcommands: pkgSubCommands},
 		{Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands},
 		{Name: "tpr2crd", Aliases: []string{}, Usage: "Migrate tool for TPR to CRD", Subcommands: migrateSubCommands},
 	}

fission/package.go

@@ -0,0 +1,325 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/urfave/cli"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/controller/client"
+	"github.com/fission/fission/crd"
+)
+
+func getFunctionsByPackage(client *client.Client, pkgName string) ([]crd.Function, error) {
+	fnList, err := client.FunctionList()
+	if err != nil {
+		return nil, err
+	}
+	fns := []crd.Function{}
+	for _, fn := range fnList {
+		if fn.Spec.Package.PackageRef.Name == pkgName {
+			fns = append(fns, fn)
+		}
+	}
+	return fns, nil
+}
+
+// downloadStoragesvcURL downloads and return archive content with given storage service url
+func downloadStoragesvcURL(client *client.Client, fileUrl string) io.ReadCloser {
+	u, err := url.ParseRequestURI(fileUrl)
+	if err != nil {
+		return nil
+	}
+	// replace in-cluster storage service host with controller server url
+	fileDownloadUrl := strings.TrimSuffix(client.Url, "/") + "/proxy/storage" + u.RequestURI()
+	reader, err := downloadURL(fileDownloadUrl)
+	checkErr(err, fmt.Sprintf("download from storage service url: %v", fileUrl))
+	return reader
+}
+
+func pkgCreate(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	envName := c.String("env")
+	if len(envName) == 0 {
+		fatal("Need --env argument.")
+	}
+
+	srcArchiveName := c.String("src")
+	deployArchiveName := c.String("deploy")
+	buildcmd := c.String("buildcmd")
+
+	if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 {
+		fatal("Need --src to specify source archive, or use --deploy to specify deployment archive.")
+	}
+
+	meta := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
+	fmt.Printf("Package '%v' created\n", meta.GetName())
+
+	return nil
+}
+
+func pkgUpdate(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fatal("Need --name argument.")
+	}
+
+	force := c.Bool("f")
+	envName := c.String("env")
+	srcArchiveName := c.String("src")
+	deployArchiveName := c.String("deploy")
+	buildcmd := c.String("buildcmd")
+
+	if len(srcArchiveName) == 0 && len(deployArchiveName) == 0 &&
+		len(envName) == 0 && len(buildcmd) == 0 {
+		fatal("Need --env or --src or --deploy or --buildcmd argument.")
+	}
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	checkErr(err, "get package")
+
+	fnList, err := getFunctionsByPackage(client, pkg.Metadata.Name)
+	checkErr(err, "get function list")
+
+	if !force && len(fnList) > 1 {
+		fatal("Package is used by multiple functions, use --force to force update")
+	}
+
+	newPkgMeta := updatePackage(client, pkg,
+		envName, srcArchiveName, deployArchiveName, buildcmd)
+
+	// update resource version of package reference of functions that shared the same package
+	for _, fn := range fnList {
+		fn.Spec.Package.PackageRef.ResourceVersion = newPkgMeta.ResourceVersion
+		_, err := client.FunctionUpdate(&fn)
+		checkErr(err, "update function")
+	}
+
+	fmt.Printf("Package '%v' updated\n", newPkgMeta.GetName())
+
+	return nil
+}
+
+func updatePackage(client *client.Client, pkg *crd.Package, envName,
+	srcArchiveName, deployArchiveName, buildcmd string) *metav1.ObjectMeta {
+
+	var srcArchiveMetadata, deployArchiveMetadata *fission.Archive
+	needToBuild := false
+
+	if len(envName) > 0 {
+		pkg.Spec.Environment.Name = envName
+		needToBuild = true
+	}
+
+	if len(buildcmd) > 0 {
+		pkg.Spec.BuildCommand = buildcmd
+		needToBuild = true
+	}
+
+	if len(srcArchiveName) > 0 {
+		srcArchiveMetadata = createArchive(client, srcArchiveName)
+		pkg.Spec.Source = *srcArchiveMetadata
+		needToBuild = true
+	}
+
+	if len(deployArchiveName) > 0 {
+		deployArchiveMetadata = createArchive(client, deployArchiveName)
+		pkg.Spec.Deployment = *deployArchiveMetadata
+	}
+
+	// Set package as pending status only when there is no
+	// deploy archive.
+	if needToBuild && len(pkg.Spec.Deployment.Type) == 0 {
+		// change into pending state to trigger package build
+		pkg.Status = fission.PackageStatus{
+			BuildStatus: fission.BuildStatusPending,
+		}
+	}
+
+	newPkgMeta, err := client.PackageUpdate(pkg)
+	checkErr(err, "update package")
+
+	return newPkgMeta
+}
+
+func pkgSourceGet(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fatal("Need name of package, use --name")
+	}
+
+	output := c.String("output")
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	var reader io.Reader
+
+	if pkg.Spec.Source.Type == fission.ArchiveTypeLiteral {
+		reader = bytes.NewReader(pkg.Spec.Source.Literal)
+	} else if pkg.Spec.Source.Type == fission.ArchiveTypeUrl {
+		readCloser := downloadStoragesvcURL(client, pkg.Spec.Source.URL)
+		defer readCloser.Close()
+		reader = readCloser
+	}
+
+	if len(output) > 0 {
+		return writeArchiveToFile(output, reader)
+	} else {
+		_, err := io.Copy(os.Stdout, reader)
+		return err
+	}
+}
+
+func pkgDeployGet(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fatal("Need name of package, use --name")
+	}
+
+	output := c.String("output")
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	var reader io.Reader
+
+	if pkg.Spec.Deployment.Type == fission.ArchiveTypeLiteral {
+		reader = bytes.NewReader(pkg.Spec.Deployment.Literal)
+	} else if pkg.Spec.Deployment.Type == fission.ArchiveTypeUrl {
+		readCloser := downloadStoragesvcURL(client, pkg.Spec.Deployment.URL)
+		defer readCloser.Close()
+		reader = readCloser
+	}
+
+	if len(output) > 0 {
+		return writeArchiveToFile(output, reader)
+	} else {
+		_, err := io.Copy(os.Stdout, reader)
+		return err
+	}
+}
+
+func pkgInfo(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fatal("Need name of package, use --name")
+	}
+
+	pkg, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\n", "Name:", pkg.Metadata.Name)
+	fmt.Fprintf(w, "%v\t%v\n", "Environment:", pkg.Spec.Environment.Name)
+	fmt.Fprintf(w, "%v\t%v\n", "Status:", pkg.Status.BuildStatus)
+	fmt.Fprintf(w, "%v\n%v", "Build Logs:", pkg.Status.BuildLog)
+	w.Flush()
+
+	return nil
+}
+
+func pkgList(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgList, err := client.PackageList()
+	if err != nil {
+		return err
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "BUILD_STATUS", "ENV")
+	for _, pkg := range pkgList {
+		fmt.Fprintf(w, "%v\t%v\t%v\n", pkg.Metadata.Name,
+			pkg.Status.BuildStatus, pkg.Spec.Environment.Name)
+	}
+	w.Flush()
+
+	return nil
+}
+
+func pkgDelete(c *cli.Context) error {
+	client := getClient(c.GlobalString("server"))
+
+	pkgName := c.String("name")
+	if len(pkgName) == 0 {
+		fmt.Println("Need --name argument.")
+		return nil
+	}
+
+	force := c.Bool("f")
+
+	_, err := client.PackageGet(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	checkErr(err, "find package")
+
+	fnList, err := getFunctionsByPackage(client, pkgName)
+
+	if !force && len(fnList) > 0 {
+		fatal("Package is used by at least one function, use -f to force delete")
+	}
+
+	err = client.PackageDelete(&metav1.ObjectMeta{
+		Namespace: metav1.NamespaceDefault,
+		Name:      pkgName,
+	})
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Package '%v' deleted\n", pkgName)
+
+	return nil
+}

glide.lock

@@ -1,5 +1,5 @@
 hash: 7c938e591b0602b9fd3412aeffd3f1438fc77eaea80c8d9dd03431adb0e1e1c9
-updated: 2017-11-01T16:18:51.144141389+08:00
+updated: 2017-12-13T16:13:45.935783+08:00
 imports:
 - name: cloud.google.com/go
   version: 3b1ae45394a234c385be014e9a488f2bb6eef821
@@ -33,7 +33,7 @@ imports:
 - name: github.com/docopt/docopt-go
   version: 784ddc588536785e7299f7272f39101f7faccc3f
 - name: github.com/dsnet/compress
-  version: c4dbed65594871659ea7347a85e1b8dcd97b9990
+  version: cc9eb1d7ad760af14e8f918698f745e80377af4f
   subpackages:
   - bzip2
   - bzip2/internal/sais
@@ -80,9 +80,9 @@ imports:
 - name: github.com/gorilla/context
   version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
 - name: github.com/gorilla/handlers
-  version: a4043c62cc2329bacda331d33fc908ab11ef0ec3
+  version: 90663712d74cb411cbef281bc1e08c19d1a76145
 - name: github.com/gorilla/mux
-  version: 24fca303ac6da784b9e8269f724ddeb0b2eea5e7
+  version: 7f08801859139f86dfafd1c296e2cba9a80d292e
 - name: github.com/graymeta/stow
   version: 1c51f76db54d79c9db24cb131011632eb586196c
   subpackages:
@@ -112,7 +112,7 @@ imports:
 - name: github.com/mholt/archiver
   version: 26cf5bb32d07aa4e8d0de15f56ce516f4641d7df
 - name: github.com/nats-io/go-nats
-  version: 9147cfac69833916f9c73353ba43f939815c330a
+  version: 6f06d34c19fc8607adf14668b5bb2c9465d78348
   subpackages:
   - encoders/builtin
   - util
@@ -121,7 +121,7 @@ imports:
   subpackages:
   - pb
 - name: github.com/nats-io/nats-streaming-server
-  version: 9ebc07951af540e177f6a5bb0424f716fd867074
+  version: f9638df9ef7ee548313eeb5a9dda9c888dc769a0
   subpackages:
   - util
 - name: github.com/nats-io/nuid

hack/release.sh

@@ -93,6 +93,24 @@ push_fetcher_image() {
     docker push $tag
 }
 
+build_builder_image() {
+    version=$1
+    tag=fission/builder:$version
+
+    pushd $DIR/builder/cmd
+
+    ./build.sh
+    docker build -t $tag .
+
+    popd
+}
+
+push_builder_image() {
+    version=$1
+    tag=fission/builder:$version
+    docker push $tag
+}
+
 build_and_push_env_image() {
     version=$1
     envdir=$2
@@ -172,6 +190,7 @@ build_all() {
     
     build_fission_bundle_image $version
     build_fetcher_image $version
+    build_builder_image $version
     build_all_cli
     build_charts $version
 }
@@ -179,6 +198,7 @@ build_all() {
 push_all() {
     push_fission_bundle_image $version
     push_fetcher_image $version
+    push_builder_image $version
 }
 
 tag_and_release() {

hack/runtests.sh

@@ -20,6 +20,6 @@ fi
 
 go test -v -i $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go')
 
-# The poolmgr unit test only works with NodePort-type services for
+# The executor unit test only works with NodePort-type services for
 # now. So disable it for our travis ci tests.
-go test -v $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v poolmgr)
+go test -v $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v executor)

hack/travis-kube-setup.sh

@@ -31,7 +31,7 @@ fi
 # Get helm
 if [ ! -f $K8SCLI_DIR/helm ]
 then
-    curl -LO https://storage.googleapis.com/kubernetes-helm/helm-v2.5.1-linux-amd64.tar.gz
+    curl -LO https://storage.googleapis.com/kubernetes-helm/helm-v2.7.2-linux-amd64.tar.gz
     tar xzvf helm-*.tar.gz
     mv linux-amd64/helm $K8SCLI_DIR/helm
 fi

hack/update-docs.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+
+set -x
+set -euo pipefail
+
+DIR=$(realpath $(dirname $0))/../
+BUILDDIR=$(realpath $DIR)/build
+DOCS_SITE=$DIR/Documentation/docs-site
+
+check_hugo() {
+    if ! which hugo
+    then
+	echo "Can't find hugo.  Go here to learn how to install it: https://gohugo.io/getting-started/installing/"
+	exit 1
+    fi
+}
+
+generate_hugo_docs() {
+    pushd $DOCS_SITE
+
+    hugo -t docdock
+    
+    popd
+}
+
+check_website_root() {
+    website_root=$1
+    pushd $website_root
+    # xxx ensure we're in the fission.io repo
+    popd
+}
+
+copy_hugo_docs() {
+    version=$1
+    website_root=$2
+    
+    check_website_root $website_root
+    pushd $website_root
+    
+    mkdir -p docs/$version
+    cp -r $DOCS_SITE/public/* docs/$version
+    git add docs/$version
+
+    pushd docs
+    rm latest
+    ln -s $version latest
+    git add latest
+    popd
+    
+    popd
+}
+
+main() {
+    version=$1
+    website_root=$2
+    
+    check_hugo
+    generate_hugo_docs
+    copy_hugo_docs $version $website_root 
+}
+
+version=$1
+website_root=$2
+main $version $website_root
+

mqtrigger/messageQueue/nats.go

@@ -88,7 +88,7 @@ func (nats Nats) unsubscribe(subscription messageQueueSubscription) error {
 
 func isTopicValidForNats(topic string) bool {
 	// nats-streaming does not support wildcard channel.
-	return nsUtil.IsSubjectValid(topic, false)
+	return nsUtil.IsChannelNameValid(topic, false)
 }
 
 func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {

poolmgr/api.go

@@ -1,253 +0,0 @@
-/*
-Copyright 2016 The Fission Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package poolmgr
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"net/http"
-	"os"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/gorilla/handlers"
-	"github.com/gorilla/mux"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/fission/fission"
-	"github.com/fission/fission/cache"
-	"github.com/fission/fission/crd"
-)
-
-type (
-	createFuncServiceRequest struct {
-		funcMeta *metav1.ObjectMeta
-		respChan chan *createFuncServiceResponse
-	}
-
-	createFuncServiceResponse struct {
-		address string
-		err     error
-	}
-
-	Poolmgr struct {
-		gpm           *GenericPoolManager
-		functionEnv   *cache.Cache // map[string]crd.Environment
-		fsCache       *functionServiceCache
-		fissionClient *crd.FissionClient
-
-		fsCreateChannels map[string]*sync.WaitGroup // xxx no channels here, rename this
-		requestChan      chan *createFuncServiceRequest
-	}
-)
-
-func MakePoolmgr(gpm *GenericPoolManager, fissionClient *crd.FissionClient, fissionNs string, fsCache *functionServiceCache) *Poolmgr {
-	poolMgr := &Poolmgr{
-		gpm:              gpm,
-		functionEnv:      cache.MakeCache(10*time.Second, 0),
-		fsCache:          fsCache,
-		fissionClient:    fissionClient,
-		fsCreateChannels: make(map[string]*sync.WaitGroup),
-		requestChan:      make(chan *createFuncServiceRequest),
-	}
-	go poolMgr.serveCreateFuncServices()
-	return poolMgr
-}
-
-// All non-cached function service requests go through this goroutine
-// serially. It parallelizes requests for different functions, and
-// ensures that for a given function, only one request causes a pod to
-// get specialized. In other words, it ensures that when there's an
-// ongoing request for a certain function, all other requests wait for
-// that request to complete.
-func (poolMgr *Poolmgr) serveCreateFuncServices() {
-	for {
-		req := <-poolMgr.requestChan
-		m := req.funcMeta
-
-		// Cache miss -- is this first one to request the func?
-		wg, found := poolMgr.fsCreateChannels[crd.CacheKey(m)]
-		if !found {
-			// create a waitgroup for other requests for
-			// the same function to wait on
-			wg := &sync.WaitGroup{}
-			wg.Add(1)
-			poolMgr.fsCreateChannels[crd.CacheKey(m)] = wg
-
-			// launch a goroutine for each request, to parallelize
-			// the specialization of different functions
-			go func() {
-				address, err := poolMgr.createServiceForFunction(m)
-				req.respChan <- &createFuncServiceResponse{
-					address: address,
-					err:     err,
-				}
-				delete(poolMgr.fsCreateChannels, crd.CacheKey(m))
-				wg.Done()
-			}()
-		} else {
-			// There's an existing request for this function, wait for it to finish
-			go func() {
-				log.Printf("Waiting for concurrent request for the same function: %v", m)
-				wg.Wait()
-
-				// get the function service from the cache
-				fsvc, err := poolMgr.fsCache.GetByFunction(m)
-				address := ""
-				if err == nil {
-					address = fsvc.address
-				}
-				req.respChan <- &createFuncServiceResponse{
-					address: address,
-					err:     err,
-				}
-			}()
-		}
-	}
-}
-
-func (poolMgr *Poolmgr) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) {
-	body, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		http.Error(w, "Failed to read request", 500)
-		return
-	}
-
-	// get function metadata
-	m := metav1.ObjectMeta{}
-	err = json.Unmarshal(body, &m)
-	if err != nil {
-		http.Error(w, "Failed to parse request", 400)
-		return
-	}
-
-	serviceName, err := poolMgr.getServiceForFunction(&m)
-	if err != nil {
-		code, msg := fission.GetHTTPError(err)
-		log.Printf("Error: %v: %v", code, msg)
-		http.Error(w, msg, code)
-		return
-	}
-
-	w.Write([]byte(serviceName))
-}
-
-func (poolMgr *Poolmgr) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment, error) {
-	var env *crd.Environment
-
-	// Cached ?
-	result, err := poolMgr.functionEnv.Get(crd.CacheKey(m))
-	if err == nil {
-		env = result.(*crd.Environment)
-		return env, nil
-	}
-
-	// Cache miss -- get func from controller
-	f, err := poolMgr.fissionClient.Functions(m.Namespace).Get(m.Name)
-	if err != nil {
-		return nil, err
-	}
-
-	// Get env from metadata
-	log.Printf("[%v] getting env", m)
-	env, err = poolMgr.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
-	if err != nil {
-		return nil, err
-	}
-
-	// cache for future lookups
-	poolMgr.functionEnv.Set(crd.CacheKey(m), env)
-
-	return env, nil
-}
-
-func (poolMgr *Poolmgr) getServiceForFunction(m *metav1.ObjectMeta) (string, error) {
-	// Check function -> svc cache
-	log.Printf("[%v] Checking for cached function service", m.Name)
-	fsvc, err := poolMgr.fsCache.GetByFunction(m)
-	if err == nil {
-		// Cached, return svc address
-		return fsvc.address, nil
-	}
-
-	respChan := make(chan *createFuncServiceResponse)
-	poolMgr.requestChan <- &createFuncServiceRequest{
-		funcMeta: m,
-		respChan: respChan,
-	}
-	resp := <-respChan
-	return resp.address, resp.err
-}
-
-func (poolMgr *Poolmgr) createServiceForFunction(m *metav1.ObjectMeta) (string, error) {
-	// None exists, so create a new funcSvc:
-	log.Printf("[%v] No cached function service found, creating one", m.Name)
-
-	// from Func -> get Env
-	log.Printf("[%v] getting environment for function", m.Name)
-	env, err := poolMgr.getFunctionEnv(m)
-	if err != nil {
-		return "", err
-	}
-
-	// from Env -> get GenericPool
-	log.Printf("[%v] getting generic pool for env", m.Name)
-	pool, err := poolMgr.gpm.GetPool(env)
-	if err != nil {
-		return "", err
-	}
-
-	// from GenericPool -> get one function container
-	// (this also adds to the cache)
-	log.Printf("[%v] getting function service from pool", m.Name)
-	fsvc, err := pool.GetFuncSvc(m)
-	if err != nil {
-		return "", err
-	}
-	return fsvc.address, nil
-}
-
-// find funcSvc and update its atime
-func (poolMgr *Poolmgr) tapService(w http.ResponseWriter, r *http.Request) {
-	body, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		http.Error(w, "Failed to read request", 500)
-		return
-	}
-	svcName := string(body)
-	svcHost := strings.TrimPrefix(svcName, "http://")
-
-	err = poolMgr.fsCache.TouchByAddress(svcHost)
-	if err != nil {
-		log.Printf("funcSvc tap error: %v", err)
-		http.Error(w, "Not found", 404)
-		return
-	}
-	w.WriteHeader(http.StatusOK)
-}
-
-func (poolMgr *Poolmgr) Serve(port int) {
-	r := mux.NewRouter()
-	r.HandleFunc("/v2/getServiceForFunction", poolMgr.getServiceForFunctionApi).Methods("POST")
-	r.HandleFunc("/v2/tapService", poolMgr.tapService).Methods("POST")
-	address := fmt.Sprintf(":%v", port)
-	log.Printf("starting poolmgr at port %v", port)
-	log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r)))
-}

poolmgr/functionServiceCache.go

@@ -1,268 +0,0 @@
-/*
-Copyright 2016 The Fission Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package poolmgr
-
-import (
-	"log"
-	"time"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/fission/fission"
-	"github.com/fission/fission/cache"
-	"github.com/fission/fission/crd"
-)
-
-type fscRequestType int
-
-const (
-	TOUCH fscRequestType = iota
-	LISTOLD
-	LOG
-	DELETE_BY_POD
-)
-
-type (
-	funcSvc struct {
-		function    *metav1.ObjectMeta // function this pod/service is for
-		environment *crd.Environment   // function's environment
-		address     string             // Host:Port or IP:Port that the function's service can be reached at.
-		podName     string             // pod name (within the function namespace)
-
-		ctime time.Time
-		atime time.Time
-	}
-
-	functionServiceCache struct {
-		byFunction *cache.Cache // function-key -> funcSvc  : map[string]*funcSvc
-		byAddress  *cache.Cache // address      -> function : map[string]metav1.ObjectMeta
-		byPod      *cache.Cache // podname      -> function : map[string]metav1.ObjectMeta
-
-		requestChannel chan *fscRequest
-	}
-	fscRequest struct {
-		requestType     fscRequestType
-		address         string
-		podName         string
-		age             time.Duration
-		env             *metav1.ObjectMeta // used for ListOld
-		responseChannel chan *fscResponse
-	}
-	fscResponse struct {
-		podNames []string
-		deleted  bool
-		error
-	}
-)
-
-func MakeFunctionServiceCache() *functionServiceCache {
-	fsc := &functionServiceCache{
-		byFunction:     cache.MakeCache(0, 0),
-		byAddress:      cache.MakeCache(0, 0),
-		byPod:          cache.MakeCache(0, 0),
-		requestChannel: make(chan *fscRequest),
-	}
-	go fsc.service()
-	return fsc
-}
-
-func (fsc *functionServiceCache) service() {
-	for {
-		req := <-fsc.requestChannel
-		resp := &fscResponse{}
-		switch req.requestType {
-		case TOUCH:
-			// update atime for this function svc
-			resp.error = fsc._touchByAddress(req.address)
-		case LISTOLD:
-			// get svcs idle for > req.age
-			byPodCopy := fsc.byPod.Copy()
-			pods := make([]string, 0)
-			for podNameI, mI := range byPodCopy {
-				m := mI.(metav1.ObjectMeta)
-				fsvcI, err := fsc.byFunction.Get(crd.CacheKey(&m))
-				if err != nil {
-					resp.error = err
-				} else {
-					fsvc := fsvcI.(*funcSvc)
-					if fsvc.environment.Metadata.UID == req.env.UID &&
-						time.Now().Sub(fsvc.atime) > req.age {
-
-						podName := podNameI.(string)
-						pods = append(pods, podName)
-					}
-				}
-			}
-			resp.podNames = pods
-		case LOG:
-			funcCopy := fsc.byFunction.Copy()
-			log.Printf("Cache has %v entries", len(funcCopy))
-			for key, fsvcI := range funcCopy {
-				fsvc := fsvcI.(*funcSvc)
-				log.Printf("%v\t%v", key, fsvc.podName)
-			}
-		case DELETE_BY_POD:
-			resp.deleted, resp.error = fsc._deleteByPod(req.podName, req.age)
-		}
-		req.responseChannel <- resp
-	}
-}
-
-func (fsc *functionServiceCache) GetByFunction(m *metav1.ObjectMeta) (*funcSvc, error) {
-	key := crd.CacheKey(m)
-
-	fsvcI, err := fsc.byFunction.Get(key)
-	if err != nil {
-		return nil, err
-	}
-
-	// update atime
-	fsvc := fsvcI.(*funcSvc)
-	fsvc.atime = time.Now()
-
-	fsvcCopy := *fsvc
-	return &fsvcCopy, nil
-}
-
-// TODO: error should be second return
-func (fsc *functionServiceCache) Add(fsvc funcSvc) (error, *funcSvc) {
-	err, existing := fsc.byFunction.Set(crd.CacheKey(fsvc.function), &fsvc)
-	if err != nil {
-		if existing != nil {
-			f := existing.(*funcSvc)
-			err2 := fsc.TouchByAddress(f.address)
-			if err2 != nil {
-				return err2, nil
-			}
-			fCopy := *f
-			return err, &fCopy
-		}
-		return err, nil
-	}
-	now := time.Now()
-	fsvc.ctime = now
-	fsvc.atime = now
-
-	// Add to byAddress and byPod caches. Ignore NameExists errors
-	// because of multiple-specialization. See issue #331.
-	err, _ = fsc.byAddress.Set(fsvc.address, *fsvc.function)
-	if err != nil {
-		if fe, ok := err.(fission.Error); ok {
-			if fe.Code == fission.ErrorNameExists {
-				err = nil
-			}
-		}
-		log.Printf("error caching fsvc: %v", err)
-		return err, nil
-	}
-	err, _ = fsc.byPod.Set(fsvc.podName, *fsvc.function)
-	if err != nil {
-		if fe, ok := err.(fission.Error); ok {
-			if fe.Code == fission.ErrorNameExists {
-				err = nil
-			}
-		}
-		log.Printf("error caching fsvc: %v", err)
-		return err, nil
-	}
-	return nil, nil
-}
-
-func (fsc *functionServiceCache) TouchByAddress(address string) error {
-	responseChannel := make(chan *fscResponse)
-	fsc.requestChannel <- &fscRequest{
-		requestType:     TOUCH,
-		address:         address,
-		responseChannel: responseChannel,
-	}
-	resp := <-responseChannel
-	return resp.error
-}
-
-func (fsc *functionServiceCache) _touchByAddress(address string) error {
-	mI, err := fsc.byAddress.Get(address)
-	if err != nil {
-		return err
-	}
-	m := mI.(metav1.ObjectMeta)
-	fsvcI, err := fsc.byFunction.Get(crd.CacheKey(&m))
-	if err != nil {
-		return err
-	}
-	fsvc := fsvcI.(*funcSvc)
-	fsvc.atime = time.Now()
-	return nil
-}
-
-func (fsc *functionServiceCache) DeleteByPod(podName string, minAge time.Duration) (bool, error) {
-	responseChannel := make(chan *fscResponse)
-	fsc.requestChannel <- &fscRequest{
-		requestType:     DELETE_BY_POD,
-		podName:         podName,
-		age:             minAge,
-		responseChannel: responseChannel,
-	}
-	resp := <-responseChannel
-	return resp.deleted, resp.error
-}
-
-// _deleteByPod deletes the entry keyed by podName, but only if it is
-// at least minAge old.
-func (fsc *functionServiceCache) _deleteByPod(podName string, minAge time.Duration) (bool, error) {
-	mI, err := fsc.byPod.Get(podName)
-	if err != nil {
-		return false, err
-	}
-	m := mI.(metav1.ObjectMeta)
-	fsvcI, err := fsc.byFunction.Get(crd.CacheKey(&m))
-	if err != nil {
-		return false, err
-	}
-	fsvc := fsvcI.(*funcSvc)
-
-	if time.Now().Sub(fsvc.atime) < minAge {
-		return false, nil
-	}
-
-	fsc.byFunction.Delete(crd.CacheKey(&m))
-	fsc.byAddress.Delete(fsvc.address)
-	fsc.byPod.Delete(podName)
-	return true, nil
-}
-
-func (fsc *functionServiceCache) ListOld(env *metav1.ObjectMeta, age time.Duration) ([]string, error) {
-	responseChannel := make(chan *fscResponse)
-	fsc.requestChannel <- &fscRequest{
-		requestType:     LISTOLD,
-		age:             age,
-		env:             env,
-		responseChannel: responseChannel,
-	}
-	resp := <-responseChannel
-	return resp.podNames, resp.error
-}
-
-func (fsc *functionServiceCache) Log() {
-	log.Printf("--- FunctionService Cache Contents")
-	responseChannel := make(chan *fscResponse)
-	fsc.requestChannel <- &fscRequest{
-		requestType:     LOG,
-		responseChannel: responseChannel,
-	}
-	<-responseChannel
-	log.Printf("--- FunctionService Cache Contents End")
-}

poolmgr/poolmgr.go

@@ -1,47 +0,0 @@
-/*
-Copyright 2016 The Fission Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package poolmgr
-
-import (
-	"log"
-
-	"github.com/dchest/uniuri"
-
-	"github.com/fission/fission/crd"
-)
-
-// Start the poolmgr service.
-func StartPoolmgr(fissionNamespace string, functionNamespace string, port int) error {
-	fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
-	if err != nil {
-		log.Printf("Failed to get kubernetes client: %v", err)
-		return err
-	}
-
-	instanceId := uniuri.NewLen(8)
-	cleanupOldPoolmgrResources(kubernetesClient, functionNamespace, instanceId)
-
-	fsCache := MakeFunctionServiceCache()
-	gpm := MakeGenericPoolManager(
-		fissionClient, kubernetesClient, fissionNamespace,
-		functionNamespace, fsCache, instanceId)
-
-	api := MakePoolmgr(gpm, fissionClient, fissionNamespace, fsCache)
-	go api.Serve(port)
-
-	return nil
-}

router/functionHandler.go

@@ -28,18 +28,18 @@ import (
 	"github.com/gorilla/mux"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	poolmgrClient "github.com/fission/fission/poolmgr/client"
+	executorClient "github.com/fission/fission/executor/client"
 )
 
 type functionHandler struct {
 	fmap     *functionServiceMap
-	poolmgr  *poolmgrClient.Client
+	executor *executorClient.Client
 	function *metav1.ObjectMeta
 }
 
 func (fh *functionHandler) getServiceForFunction() (*url.URL, error) {
-	// call poolmgr, get a url for a function
+	// call executor, get a url for a function
-	svcName, err := fh.poolmgr.GetServiceForFunction(fh.function)
+	svcName, err := fh.executor.GetServiceForFunction(fh.function)
 	if err != nil {
 		return nil, err
 	}
@@ -83,10 +83,10 @@ func (rrt RetryingRoundTripper) RoundTrip(req *http.Request) (*http.Response, er
 }
 
 func (fh *functionHandler) tapService(serviceUrl *url.URL) {
-	if fh.poolmgr == nil {
+	if fh.executor == nil {
 		return
 	}
-	fh.poolmgr.TapService(serviceUrl)
+	fh.executor.TapService(serviceUrl)
 }
 
 func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *http.Request) {
@@ -121,7 +121,7 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *
 		fh.fmap.assign(fh.function, serviceUrl)
 	} else {
 		// if we're using our cache, asynchronously tell
-		// poolmgr we're using this service
+		// executor we're using this service
 		go fh.tapService(serviceUrl)
 	}
 
@@ -141,6 +141,11 @@ func (fh *functionHandler) handler(responseWriter http.ResponseWriter, request *
 		// function metadata here.
 		req.URL.Path = "/"
 
+		// Overwrite request host with internal host,
+		// or request will be blocked in some situations
+		// (e.g. istio-proxy)
+		req.Host = serviceUrl.Host
+
 		// leave the query string intact (req.URL.RawQuery)
 
 		if _, ok := req.Header["User-Agent"]; !ok {

router/httpTriggers.go

@@ -30,14 +30,14 @@ import (
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/crd"
-	poolmgrClient "github.com/fission/fission/poolmgr/client"
+	executorClient "github.com/fission/fission/executor/client"
 )
 
 type HTTPTriggerSet struct {
 	*functionServiceMap
 	*mutableRouter
 	fissionClient     *crd.FissionClient
-	poolmgr           *poolmgrClient.Client
+	executor          *executorClient.Client
 	resolver          *functionReferenceResolver
 	crdClient         *rest.RESTClient
 	triggers          []crd.HTTPTrigger
@@ -49,12 +49,12 @@ type HTTPTriggerSet struct {
 }
 
 func makeHTTPTriggerSet(fmap *functionServiceMap, fissionClient *crd.FissionClient,
-	poolmgr *poolmgrClient.Client, crdClient *rest.RESTClient) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
+	executor *executorClient.Client, crdClient *rest.RESTClient) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
 	httpTriggerSet := &HTTPTriggerSet{
 		functionServiceMap: fmap,
 		triggers:           []crd.HTTPTrigger{},
 		fissionClient:      fissionClient,
-		poolmgr:            poolmgr,
+		executor:           executor,
 		crdClient:          crdClient,
 	}
 	var tStore, fnStore k8sCache.Store
@@ -114,9 +114,14 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 		fh := &functionHandler{
 			fmap:     ts.functionServiceMap,
 			function: rr.functionMetadata,
-			poolmgr:  ts.poolmgr,
+			executor: ts.executor,
+		}
+
+		ht := muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler)
+		ht.Methods(trigger.Spec.Method)
+		if trigger.Spec.Host != "" {
+			ht.Host(trigger.Spec.Host)
 		}
-		muxRouter.HandleFunc(trigger.Spec.RelativeURL, fh.handler).Methods(trigger.Spec.Method)
 		if trigger.Spec.RelativeURL == "/" && trigger.Spec.Method == "GET" {
 			homeHandled = true
 		}
@@ -139,7 +144,7 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
 		fh := &functionHandler{
 			fmap:     ts.functionServiceMap,
 			function: &m,
-			poolmgr:  ts.poolmgr,
+			executor: ts.executor,
 		}
 		muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler)
 	}

router/router.go

@@ -51,7 +51,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/fission/fission/crd"
-	poolmgrClient "github.com/fission/fission/poolmgr/client"
+	executorClient "github.com/fission/fission/executor/client"
 )
 
 // request url ---[mux]---> Function(name,uid) ----[fmap]----> k8s service url
@@ -71,17 +71,20 @@ func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolv
 	http.ListenAndServe(url, handlers.LoggingHandler(os.Stdout, mr))
 }
 
-func Start(port int, poolmgrUrl string) {
+func Start(port int, executorUrl string) {
 	fmap := makeFunctionServiceMap(time.Minute)
 
 	fissionClient, _, _, err := crd.MakeFissionClient()
 	if err != nil {
 		log.Fatalf("Error connecting to kubernetes API: %v", err)
 	}
+
 	restClient := fissionClient.GetCrdClient()
-	poolmgr := poolmgrClient.MakeClient(poolmgrUrl)
+
-	triggers, _, fnStore := makeHTTPTriggerSet(fmap, fissionClient, poolmgr, restClient)
+	executor := executorClient.MakeClient(executorUrl)
+	triggers, _, fnStore := makeHTTPTriggerSet(fmap, fissionClient, executor, restClient)
 	resolver := makeFunctionReferenceResolver(fnStore)
+
 	log.Printf("Starting router at port %v\n", port)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

test/build_and_test.sh

@@ -14,19 +14,22 @@ REPO=gcr.io/fission-ci
 IMAGE=$REPO/fission-bundle
 FETCHER_IMAGE=$REPO/fetcher
 FLUENTD_IMAGE=gcr.io/fission-ci/fluentd
+BUILDER_IMAGE=$REPO/builder
 TAG=test
 
+dump_system_info
+
 build_and_push_fission_bundle $IMAGE:$TAG
 
 build_and_push_fetcher $FETCHER_IMAGE:$TAG
 
-build_builder
+build_and_push_builder $BUILDER_IMAGE:$TAG
 
 ENV='python'
 
 build_and_push_env_runtime $ENV $REPO/$ENV-env:$TAG
 
-build_and_push_env_builder $ENV $REPO/$ENV-env-builder:$TAG
+build_and_push_env_builder $ENV $REPO/$ENV-env-builder:$TAG $BUILDER_IMAGE:$TAG
 
 build_and_push_fluentd $FLUENTD_IMAGE:$TAG
 

test/test_utils.sh

@@ -8,11 +8,41 @@
 
 set -euo pipefail
 
-ROOT=$(dirname $0)/..
+ROOT_RELPATH=$(dirname $0)/..
+pushd $ROOT_RELPATH
+ROOT=$(pwd)
+popd
+
+export TEST_REPORT=""
+
+report_msg() {
+    TEST_REPORT="$TEST_REPORT\n$1"
+}
+report_test_passed() {
+    report_msg "--- PASSED $1"
+}
+report_test_failed() {
+    report_msg "*** FAILED $1"
+}
+report_test_skipped() {
+    report_msg "### SKIPPED $1"
+}
+show_test_report() {
+    echo -e "------\n$TEST_REPORT\n------"
+}
 
 helm_setup() {
     helm init
+    # wait for tiller ready
+    while true; do
+      kubectl --namespace kube-system get pod|grep tiller|grep Running
+      if [[ $? -eq 0 ]]; then
+          break
+      fi
+      sleep 1
+    done
 }
+export -f helm_setup
 
 gcloud_login() {
     KEY=${HOME}/gcloud-service-key.json
@@ -50,9 +80,17 @@ build_and_push_fetcher() {
     popd
 }
 
-build_builder() {
+
+build_and_push_builder() {
+    image_tag=$1
+
     pushd $ROOT/builder/cmd
     ./build.sh
+    docker build -t $image_tag .
+
+    gcloud_login
+
+    gcloud docker -- push $image_tag
     popd
 }
 
@@ -85,12 +123,11 @@ build_and_push_env_runtime() {
 build_and_push_env_builder() {
     env=$1
     image_tag=$2
+    builder_image=$3
 
     pushd $ROOT/environments/$env/builder
-    builderDir=${GOPATH}/src/github.com/fission/fission/builder/cmd
-    cp ${builderDir}/builder .
 
-    docker build -t $image_tag .
+    docker build -t $image_tag --build-arg BUILDER_IMAGE=${builder_image} .
 
     gcloud_login
 
@@ -130,10 +167,10 @@ helm_install_fission() {
 
     helmVars=image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImage=$fluentdImage
 
-    helm_setup
+    timeout 30 bash -c "helm_setup"
 
-    echo "Deleting failed releases"
+    echo "Deleting old releases"
-    helm list --failed -q|xargs -I@ bash -c "helm delete @"
+    helm list -q|xargs -I@ bash -c "helm_uninstall_fission @"
 
     echo "Installing fission"
     helm install		\
@@ -172,15 +209,20 @@ wait_for_services() {
     wait_for_service $id router
 }
 
-helm_uninstall_fission() {
+helm_uninstall_fission() {(set +e
+    id=$1
+
     if [ ! -z ${FISSION_TEST_SKIP_DELETE:+} ]
     then
 	echo "Fission uninstallation skipped"
 	return
     fi
     echo "Uninstalling fission"
-    helm delete --purge $1
+    helm delete --purge $id
-}
+
+    kubectl delete ns f-$id
+)}
+export -f helm_uninstall_fission
 
 set_environment() {
     id=$1
@@ -230,10 +272,13 @@ dump_fission_crd() {
 }
 
 dump_fission_crds() {
-    dump_fission_crd function.fission.io    
+    dump_fission_crd environments.fission.io
-    dump_fission_crd package.fission.io    
+    dump_fission_crd functions.fission.io
-    dump_fission_crd httptrigger.fission.io    
+    dump_fission_crd httptriggers.fission.io
-    dump_fission_crd environment.fission.io    
+    dump_fission_crd kuberneteswatchtriggers.fission.io
+    dump_fission_crd messagequeuetriggers.fission.io
+    dump_fission_crd packages.fission.io
+    dump_fission_crd timetriggers.fission.io
 }
 
 dump_env_pods() {
@@ -252,6 +297,15 @@ dump_all_fission_resources() {
     echo "--- End objects in the fission namespace $ns ---"
 }
 
+dump_system_info() {
+    echo "--- System Info ---"
+    go version
+    docker version
+    kubectl version
+    helm version
+    echo "--- End System Info ---"
+}
+
 dump_logs() {
     id=$1
 
@@ -262,8 +316,8 @@ dump_logs() {
     dump_env_pods $fns
     dump_fission_logs $ns $fns controller
     dump_fission_logs $ns $fns router
-    dump_fission_logs $ns $fns poolmgr
     dump_fission_logs $ns $fns buildermgr
+    dump_fission_logs $ns $fns executor
     dump_function_pod_logs $ns $fns
     dump_fission_crds
 }
@@ -276,15 +330,30 @@ run_all_tests() {
     export FISSION_NAMESPACE=f-$id
     export FUNCTION_NAMESPACE=f-func-$id
 
-    for file in $ROOT/test/tests/test_*.sh
+    test_files=$(find $ROOT/test/tests -iname 'test_*.sh')
+
+    for file in $test_files
     do
-	echo ------- Running $file -------
+	testname=${file#$ROOT/test/tests}
-	if $file
+	testpath=$file
+
+	if grep "^#test:disabled" $file
 	then
-	    echo SUCCESS: $file
+	    report_test_skipped $testname
+	    echo ------- Skipped $testname -------
 	else
-	    echo FAILED: $file
+	    echo ------- Running $testname -------
-	    export FAILURES=$(($FAILURES+1))
+	    pushd $(dirname $testpath)
+	    if $testpath
+	    then
+		echo SUCCESS: $testname
+		report_test_passed $testname
+	    else
+		echo FAILED: $testname
+		export FAILURES=$(($FAILURES+1))
+		report_test_failed $testname
+	    fi
+	    popd
 	fi
     done
 }
@@ -317,6 +386,8 @@ install_and_test() {
 
     dump_logs $id
 
+    show_test_report
+
     if [ $FAILURES -ne 0 ]
     then
 	exit 1

test/tests/test_logging/log.js

@@ -1,8 +1,8 @@
 
 module.exports = async function(context) {
-    console.log("log test log test log test")
+    console.log("log test")
     return {
         status: 200,
-        body: "Log, test!\n"
+        body: "log test\n"
     };
 }

test/tests/test_logging/test_function_logs.sh

@@ -1,17 +1,18 @@
 #!/bin/bash
 
+
 set -euo pipefail
 
 ROOT=$(dirname $0)/../..
 
-fn=nodejs-logtest
+fn=nodejs-logtest-$(date +%N)
-
 
 function cleanup {
     echo "Cleanup route"
     var=$(fission route list | grep $fn | awk '{print $1;}')
     fission route delete --name $var
-    fission function delete --name $fn
+    echo "delete logfile"
+    rm "/tmp/logfile"
 }
 
 # Create a hello world function in nodejs, test it with an http trigger
@@ -27,32 +28,39 @@ fission fn create --name $fn --env nodejs --code log.js
 trap "fission fn delete --name $fn" EXIT
 
 echo "Creating route"
-fission route create --function $fn --url /logtest --method GET
+fission route create --function $fn --url /$fn --method GET
+trap cleanup EXIT
 
 echo "Waiting for router to catch up"
-sleep 3
+sleep 15
-
-echo "Doing 4 HTTP GET on the function's route"
-curl http://$FISSION_ROUTER/logtest
-curl http://$FISSION_ROUTER/logtest
-curl http://$FISSION_ROUTER/logtest
-curl http://$FISSION_ROUTER/logtest
 
+echo "Doing 4 HTTP GETs on the function's route"
+for i in 1 2 3 4
+do
+    curl -s http://$FISSION_ROUTER/$fn
+done
 
 echo "Grabbing logs, should have 4 calls in logs"
 
 sleep 15
 
-echo "woke up"
+fission function logs --name $fn --detail > /tmp/logfile
-logs=$(fission function logs --name $fn)
+
-num=$(echo "$logs" | grep 'log test' | wc -l)
+size=$(wc -c </tmp/logfile)
-echo $num
+if [ $size = 0 ]
+then
+    fission function logs --name $fn --detail > /tmp/logfile
+fi
+
+echo "---function logs---"
+cat /tmp/logfile
+echo "------"
+num=$(grep 'log test' /tmp/logfile | wc -l)
+echo $num logs found
 
 if [ $num -ne 4 ]
 then
-    echo "Test Failed"
+    echo "Test Failed: expected 4, found $num logs"
-    trap cleanup EXIT
 fi
-cleanup
  
 echo "All done."

test/tests/test_package_command.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Use package command to create two packages one with source 
+# archive and the other with deploy archive. Also, create a 
+# function to test the packages created by package command are 
+# able to work.
+
+ROOT=$(dirname $0)/../..
+PYTHON_RUNTIME_IMAGE=gcr.io/fission-ci/python3-env:test
+PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python3-env-builder:test
+
+fn=python-srcbuild-$(date +%s)
+
+waitBuild() {
+    echo "Waiting for builder manager to finish the build"
+    
+    while true; do
+      kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded
+      if [[ $? -eq 0 ]]; then
+          break
+      fi
+    done
+}
+export -f waitBuild
+
+checkFunctionResponse() {
+    echo "Doing an HTTP GET on the function's route"
+    response=$(curl http://$FISSION_ROUTER/$1)
+
+    echo "Checking for valid response"
+    echo $response
+    echo $response | grep -i "$2"
+}
+
+echo "Pre-test cleanup"
+fission env delete --name python || true
+
+echo "Creating python env"
+fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
+trap "fission env delete --name python" EXIT
+
+echo "Waiting for env builder to catch up"
+sleep 30
+
+echo "Creating pacakage with source archive"
+zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
+pkgName=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
+
+# wait for build to finish at most 60s
+timeout 60s bash -c "waitBuild $pkgName"
+
+echo "Creating function " $fn
+fission fn create --name $fn --pkg $pkgName --entrypoint "user.main"
+trap "fission fn delete --name $fn" EXIT
+
+echo "Creating route"
+fission route create --function $fn --url /$fn --method GET
+
+echo "Waiting for router to catch up"
+sleep 3
+  
+checkFunctionResponse $fn 'a: 1 b: {c: 3, d: 4}'
+
+echo "Creating package with deploy archive"
+mkdir testDir
+touch testDir/__init__.py
+printf 'def main():\n    return "Hello, world!"' > testDir/hello.py
+zip -jr demo-deploy-pkg.zip testDir/
+pkgName=$(fission package create --deploy demo-deploy-pkg.zip --env python| cut -f2 -d' '| tr -d \')
+
+echo "Updating function " $fn
+fission fn update --name $fn --pkg $pkgName --entrypoint "hello.main"
+trap "fission fn delete --name $fn" EXIT
+
+echo "Waiting for router to update cache"
+sleep 3
+
+checkFunctionResponse $fn 'Hello, world!'
+
+# crappy cleanup, improve this later
+kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
+
+echo "All done."

types.go

@@ -151,7 +151,10 @@ type (
 		FunctionEndpointPort int32 `json:"functionendpointport"`
 	}
 	Builder struct {
-		Image   string `json:"image"`
+		// Image for containing the language runtime.
+		Image string `json:"image"`
+
+		// (Optional) Default build command to run for this build environment.
 		Command string `json:"command"`
 	}
 	EnvironmentSpec struct {