Move packages to proejct/pkg to follow go project folder structure convention (#1190)

cmd/builder/Dockerfile.fission-builder

@@ -0,0 +1,26 @@
+FROM golang:1.11-alpine as fission-builder
+RUN apk add bash ca-certificates git gcc g++ libc-dev
+
+ARG GITCOMMIT=unknown
+# E.g. GITCOMMIT=$(git rev-parse HEAD)
+
+ARG BUILDVERSION=unknown
+# E.g. BUILDVERSION=$(git rev-parse HEAD)
+
+ARG BUILDDATE=unknown
+# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+
+ARG GOPKG=github.com/fission/fission
+COPY . /go/src/${GOPKG}
+WORKDIR /go/src/${GOPKG}/cmd/builder
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
+    -o /go/bin/builder \
+    -gcflags=-trimpath=$GOPATH \
+    -asmflags=-trimpath=$GOPATH \
+    -ldflags "-X github.com/fission/fission/pkg/info.GitCommit=${GITCOMMIT} -X github.com/fission/fission/pkg/info.BuildDate=${BUILDDATE} -X github.com/fission/fission/pkg/info.Version=${BUILDVERSION}"
+
+FROM alpine:3.5
+COPY --from=fission-builder /go/bin/builder /
+EXPOSE 8001
+
+ENTRYPOINT ["/builder"]

cmd/builder/app/server.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"net/http"
+
+	"go.uber.org/zap"
+
+	builder "github.com/fission/fission/pkg/builder"
+)
+
+// Usage: builder <shared volume path>
+func Run(logger *zap.Logger, shareVolume string) error {
+	builder := builder.MakeBuilder(logger, shareVolume)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", builder.Handler)
+	mux.HandleFunc("/version", builder.VersionHandler)
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+	return http.ListenAndServe(":8001", mux)
+}

cmd/builder/main.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"log"
+	"os"
+
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/cmd/builder/app"
+)
+
+// Usage: builder <shared volume path>
+func main() {
+	logger, err := zap.NewProduction()
+	if err != nil {
+		log.Fatalf("can't initialize zap logger: %v", err)
+	}
+	defer logger.Sync()
+
+	shareVolume := os.Args[1]
+	if _, err := os.Stat(shareVolume); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(shareVolume, os.ModeDir|0700)
+			if err != nil {
+				logger.Fatal("error creating directory: %v", zap.Error(err), zap.String("directory", shareVolume))
+			}
+		}
+	}
+
+	err = app.Run(logger, shareVolume)
+	logger.Error("error running builder", zap.Error(err))
+}

cmd/fetcher/Dockerfile.fission-fetcher

@@ -0,0 +1,26 @@
+FROM golang:1.11-alpine as builder
+RUN apk add bash ca-certificates git gcc g++ libc-dev
+
+ARG GITCOMMIT=unknown
+# E.g. GITCOMMIT=$(git rev-parse HEAD)
+
+ARG BUILDVERSION=unknown
+# E.g. BUILDVERSION=$(git rev-parse HEAD)
+
+ARG BUILDDATE=unknown
+# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+
+ARG GOPKG=github.com/fission/fission
+COPY . /go/src/${GOPKG}
+WORKDIR /go/src/${GOPKG}/cmd/fetcher
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
+    -o /go/bin/fetcher \
+    -gcflags=-trimpath=$GOPATH \
+    -asmflags=-trimpath=$GOPATH \
+    -ldflags "-X github.com/fission/fission/pkg/info.GitCommit=${GITCOMMIT} -X github.com/fission/fission/pkg/info.BuildDate=${BUILDDATE} -X github.com/fission/fission/pkg/info.Version=${BUILDVERSION}"
+
+FROM alpine:3.4
+COPY --from=builder /go/bin/fetcher /
+EXPOSE 8000
+
+ENTRYPOINT ["/fetcher"]

cmd/fetcher/app/server.go

@@ -0,0 +1,122 @@
+package app
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+
+	"go.opencensus.io/exporter/jaeger"
+	"go.opencensus.io/plugin/ochttp"
+	"go.opencensus.io/trace"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/fetcher"
+	"github.com/fission/fission/pkg/types"
+)
+
+func registerTraceExporter(collectorEndpoint string) error {
+	if collectorEndpoint == "" {
+		return nil
+	}
+
+	serviceName := "Fission-Fetcher"
+	exporter, err := jaeger.NewExporter(jaeger.Options{
+		CollectorEndpoint: collectorEndpoint,
+		Process: jaeger.Process{
+			ServiceName: serviceName,
+			Tags: []jaeger.Tag{
+				jaeger.BoolTag("fission", true),
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+	trace.RegisterExporter(exporter)
+	trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
+	return nil
+}
+
+func Run(logger *zap.Logger) {
+	flag.Usage = fetcherUsage
+	collectorEndpoint := flag.String("jaeger-collector-endpoint", "", "")
+	specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup")
+	specializePayload := flag.String("specialize-request", "", "JSON payload for specialize request")
+	secretDir := flag.String("secret-dir", "", "Path to shared secrets directory")
+	configDir := flag.String("cfgmap-dir", "", "Path to shared configmap directory")
+
+	flag.Parse()
+	if flag.NArg() == 0 {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	dir := flag.Arg(0)
+	if _, err := os.Stat(dir); err != nil {
+		if os.IsNotExist(err) {
+			err = os.MkdirAll(dir, os.ModeDir|0700)
+			if err != nil {
+				logger.Fatal("error creating directory", zap.Error(err), zap.String("directory", dir))
+			}
+		}
+	}
+
+	if err := registerTraceExporter(*collectorEndpoint); err != nil {
+		logger.Fatal("could not register trace exporter", zap.Error(err), zap.String("collector_endpoint", *collectorEndpoint))
+	}
+
+	f, err := fetcher.MakeFetcher(logger, dir, *secretDir, *configDir)
+	if err != nil {
+		logger.Fatal("error making fetcher", zap.Error(err))
+	}
+
+	readyToServe := false
+
+	// do specialization in other goroutine to prevent blocking in newdeploy
+	go func() {
+		if *specializeOnStart {
+			var specializeReq types.FunctionSpecializeRequest
+
+			err := json.Unmarshal([]byte(*specializePayload), &specializeReq)
+			if err != nil {
+				logger.Fatal("error decoding specialize request", zap.Error(err))
+			}
+
+			ctx := context.Background()
+			err = f.SpecializePod(ctx, specializeReq.FetchReq, specializeReq.LoadReq)
+			if err != nil {
+				logger.Fatal("error specializing function pod", zap.Error(err))
+			}
+
+			readyToServe = true
+		}
+	}()
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/fetch", f.FetchHandler)
+	mux.HandleFunc("/specialize", f.SpecializeHandler)
+	mux.HandleFunc("/upload", f.UploadHandler)
+	mux.HandleFunc("/version", f.VersionHandler)
+	mux.HandleFunc("/readniess-healthz", func(w http.ResponseWriter, r *http.Request) {
+		if !*specializeOnStart || readyToServe {
+			w.WriteHeader(http.StatusOK)
+		} else {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		}
+	})
+	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	})
+
+	logger.Info("fetcher ready to receive requests")
+	http.ListenAndServe(":8000", &ochttp.Handler{
+		Handler: mux,
+	})
+}
+
+func fetcherUsage() {
+	fmt.Println("Usage: fetcher [-specialize-on-startup] [-specialize-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path>")
+}

cmd/fetcher/main.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"log"
+
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/cmd/fetcher/app"
+)
+
+// Usage: fetcher <shared volume path>
+func main() {
+	logger, err := zap.NewProduction()
+	if err != nil {
+		log.Fatalf("can't initialize zap logger: %v", err)
+	}
+	defer logger.Sync()
+
+	app.Run(logger)
+}

cmd/fission-bundle/Dockerfile.fission-bundle

@@ -0,0 +1,28 @@
+FROM golang:1.11-alpine as builder
+RUN apk add bash ca-certificates git gcc g++ libc-dev
+
+ARG GITCOMMIT=unknown
+# E.g. GITCOMMIT=$(git rev-parse HEAD)
+
+ARG BUILDVERSION=unknown
+# E.g. BUILDVERSION=$(git rev-parse HEAD)
+
+ARG BUILDDATE=unknown
+# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+
+ARG GOPKG=github.com/fission/fission
+COPY . /go/src/${GOPKG}
+RUN rm -f /go/src/${GOPKG}/Dockerfile*
+WORKDIR /go/src/${GOPKG}/cmd/fission-bundle
+ENV GO111MODULE=on
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -mod vendor \
+    -o /go/bin/fission-bundle \
+    -gcflags=-trimpath=$GOPATH \
+    -asmflags=-trimpath=$GOPATH \
+    -ldflags "-X github.com/fission/fission/pkg/info.GitCommit=${GITCOMMIT} -X github.com/fission/fission/pkg/info.BuildDate=${BUILDDATE} -X github.com/fission/fission/pkg/info.Version=${BUILDVERSION}"
+
+FROM alpine:3.4
+RUN apk add --update ca-certificates
+COPY --from=builder /go/bin/fission-bundle /
+
+ENTRYPOINT ["/fission-bundle"]

cmd/fission-bundle/main.go

@@ -0,0 +1,278 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+
+	docopt "github.com/docopt/docopt-go"
+	"go.opencensus.io/exporter/jaeger"
+	"go.opencensus.io/trace"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/buildermgr"
+	"github.com/fission/fission/pkg/controller"
+	"github.com/fission/fission/pkg/executor"
+	"github.com/fission/fission/pkg/info"
+	"github.com/fission/fission/pkg/kubewatcher"
+	functionLogger "github.com/fission/fission/pkg/logger"
+	messagequeue "github.com/fission/fission/pkg/mqtrigger"
+	"github.com/fission/fission/pkg/router"
+	"github.com/fission/fission/pkg/storagesvc"
+	"github.com/fission/fission/pkg/timer"
+)
+
+func runController(logger *zap.Logger, port int) {
+	controller.Start(logger, port, false)
+	logger.Fatal("controller exited")
+}
+
+func runRouter(logger *zap.Logger, port int, executorUrl string) {
+	router.Start(logger, port, executorUrl)
+	logger.Fatal("router exited")
+}
+
+func runExecutor(logger *zap.Logger, port int, fissionNamespace, functionNamespace, envBuilderNamespace string) {
+	err := executor.StartExecutor(logger, fissionNamespace, functionNamespace, envBuilderNamespace, port)
+	if err != nil {
+		logger.Fatal("error starting executor", zap.Error(err))
+	}
+}
+
+func runKubeWatcher(logger *zap.Logger, routerUrl string) {
+	err := kubewatcher.Start(logger, routerUrl)
+	if err != nil {
+		logger.Fatal("error starting kubewatcher", zap.Error(err))
+	}
+}
+
+func runTimer(logger *zap.Logger, routerUrl string) {
+	err := timer.Start(logger, routerUrl)
+	if err != nil {
+		logger.Fatal("error starting timer", zap.Error(err))
+	}
+}
+
+func runMessageQueueMgr(logger *zap.Logger, routerUrl string) {
+	err := messagequeue.Start(logger, routerUrl)
+	if err != nil {
+		logger.Fatal("error starting message queue manager", zap.Error(err))
+	}
+}
+
+func runStorageSvc(logger *zap.Logger, port int, filePath string) {
+	subdir := os.Getenv("SUBDIR")
+	if len(subdir) == 0 {
+		subdir = "fission-functions"
+	}
+	enableArchivePruner := true
+	storagesvc.RunStorageService(logger, storagesvc.StorageTypeLocal,
+		filePath, subdir, port, enableArchivePruner)
+}
+
+func runBuilderMgr(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) {
+	err := buildermgr.Start(logger, storageSvcUrl, envBuilderNamespace)
+	if err != nil {
+		logger.Fatal("error starting builder manager", zap.Error(err))
+	}
+}
+
+func runLogger() {
+	functionLogger.Start()
+	log.Fatalf("Error: Logger exited.")
+}
+
+func getPort(logger *zap.Logger, portArg interface{}) int {
+	portArgStr := portArg.(string)
+	port, err := strconv.Atoi(portArgStr)
+	if err != nil {
+		logger.Fatal("invalid port number", zap.Error(err), zap.String("port", portArgStr))
+	}
+	return port
+}
+
+func getStringArgWithDefault(arg interface{}, defaultValue string) string {
+	if arg != nil {
+		return arg.(string)
+	} else {
+		return defaultValue
+	}
+}
+
+func registerTraceExporter(logger *zap.Logger, arguments map[string]interface{}) error {
+	collectorEndpoint := getStringArgWithDefault(arguments["--collectorEndpoint"], "")
+	if collectorEndpoint == "" {
+		logger.Info("skipping trace exporter registration")
+		return nil
+	}
+
+	serviceName := "Fission-Unknown"
+
+	if arguments["--controllerPort"] != nil {
+		serviceName = "Fission-Controller"
+	} else if arguments["--routerPort"] != nil {
+		serviceName = "Fission-Router"
+	} else if arguments["--executorPort"] != nil {
+		serviceName = "Fission-Executor"
+	} else if arguments["--kubewatcher"] == true {
+		serviceName = "Fission-KubeWatcher"
+	} else if arguments["--timer"] == true {
+		serviceName = "Fission-Timer"
+	} else if arguments["--mqt"] == true {
+		serviceName = "Fission-MessageQueueTrigger"
+	} else if arguments["--builderMgr"] == true {
+		serviceName = "Fission-BuilderMgr"
+	} else if arguments["--storageServicePort"] != nil {
+		serviceName = "Fission-StorageSvc"
+	}
+
+	exporter, err := jaeger.NewExporter(jaeger.Options{
+		CollectorEndpoint: collectorEndpoint,
+		Process: jaeger.Process{
+			ServiceName: serviceName,
+			Tags: []jaeger.Tag{
+				jaeger.BoolTag("fission", true),
+			},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	samplingRate, err := strconv.ParseFloat(os.Getenv("TRACING_SAMPLING_RATE"), 32)
+	if err != nil {
+		return err
+	}
+	trace.RegisterExporter(exporter)
+	trace.ApplyConfig(trace.Config{DefaultSampler: trace.ProbabilitySampler(samplingRate)})
+	return nil
+}
+
+func main() {
+	// From https://github.com/containous/traefik/pull/1817/files
+	// Tell glog to log into STDERR. Otherwise, we risk
+	// certain kinds of API errors getting logged into a directory not
+	// available in a `FROM scratch` Docker container, causing glog to abort
+	// hard with an exit code > 0.
+	flag.Set("logtostderr", "true")
+
+	usage := `fission-bundle: Package of all fission microservices: controller, router, executor.
+
+Use it to start one or more of the fission servers:
+
+ Controller is a stateless API frontend for fission resources.
+
+ Pool manager maintains a pool of generalized function containers, and
+ specializes them on-demand. Executor must be run from a pod in a
+ Kubernetes cluster.
+
+ Router implements HTTP triggers: it routes to running instances,
+ working with the controller and executor.
+
+ Kubewatcher implements Kubernetes Watch triggers: it watches
+ Kubernetes resources and invokes functions described in the
+ KubernetesWatchTrigger.
+
+ The storage service implements storage for functions too large to fit
+ in the Kubernetes API resource object. It supports various storage
+ backends.
+
+Usage:
+  fission-bundle --controllerPort=<port> [--collectorEndpoint=<url>]
+  fission-bundle --routerPort=<port> [--executorUrl=<url>] [--collectorEndpoint=<url>]
+  fission-bundle --executorPort=<port> [--namespace=<namespace>] [--fission-namespace=<namespace>] [--collectorEndpoint=<url>]
+  fission-bundle --kubewatcher [--routerUrl=<url>] [--collectorEndpoint=<url>]
+  fission-bundle --storageServicePort=<port> --filePath=<filePath> [--collectorEndpoint=<url>]
+  fission-bundle --builderMgr [--storageSvcUrl=<url>] [--envbuilder-namespace=<namespace>] [--collectorEndpoint=<url>]
+  fission-bundle --timer [--routerUrl=<url>] [--collectorEndpoint=<url>]
+  fission-bundle --mqt   [--routerUrl=<url>] [--collectorEndpoint=<url>]
+  fission-bundle --logger
+  fission-bundle --version
+Options:
+  --collectorEndpoint=<url> Jaeger HTTP Thrift collector URL.
+  --controllerPort=<port>         Port that the controller should listen on.
+  --routerPort=<port>             Port that the router should listen on.
+  --executorPort=<port>           Port that the executor should listen on.
+  --storageServicePort=<port>     Port that the storage service should listen on.
+  --executorUrl=<url>             Executor URL. Not required if --executorPort is specified.
+  --routerUrl=<url>               Router URL.
+  --etcdUrl=<etcdUrl>             Etcd URL.
+  --storageSvcUrl=<url>           StorageService URL.
+  --filePath=<filePath>           Directory to store functions in.
+  --namespace=<namespace>         Kubernetes namespace in which to run function containers. Defaults to 'fission-function'.
+  --kubewatcher                   Start Kubernetes events watcher.
+  --timer                         Start Timer.
+  --mqt                           Start message queue trigger.
+  --builderMgr                    Start builder manager.
+  --version                       Print version information
+`
+	logger, err := zap.NewProduction()
+	if err != nil {
+		log.Fatalf("can't initialize zap logger: %v", err)
+	}
+	defer logger.Sync()
+
+	version := fmt.Sprintf("Fission Bundle Version: %v", info.BuildInfo().String())
+	arguments, err := docopt.Parse(usage, nil, true, version, false)
+	if err != nil {
+		logger.Fatal("Could not parse command line arguments", zap.Error(err))
+	}
+
+	err = registerTraceExporter(logger, arguments)
+	if err != nil {
+		logger.Fatal("Could not register trace exporter", zap.Error(err), zap.Any("argument", arguments))
+	}
+
+	functionNs := getStringArgWithDefault(arguments["--namespace"], "fission-function")
+	fissionNs := getStringArgWithDefault(arguments["--fission-namespace"], "fission")
+	envBuilderNs := getStringArgWithDefault(arguments["--envbuilder-namespace"], "fission-builder")
+
+	executorUrl := getStringArgWithDefault(arguments["--executorUrl"], "http://executor.fission")
+	routerUrl := getStringArgWithDefault(arguments["--routerUrl"], "http://router.fission")
+	storageSvcUrl := getStringArgWithDefault(arguments["--storageSvcUrl"], "http://storagesvc.fission")
+
+	if arguments["--controllerPort"] != nil {
+		port := getPort(logger, arguments["--controllerPort"])
+		runController(logger, port)
+	}
+
+	if arguments["--routerPort"] != nil {
+		port := getPort(logger, arguments["--routerPort"])
+		runRouter(logger, port, executorUrl)
+	}
+
+	if arguments["--executorPort"] != nil {
+		port := getPort(logger, arguments["--executorPort"])
+		runExecutor(logger, port, fissionNs, functionNs, envBuilderNs)
+	}
+
+	if arguments["--kubewatcher"] == true {
+		runKubeWatcher(logger, routerUrl)
+	}
+
+	if arguments["--timer"] == true {
+		runTimer(logger, routerUrl)
+	}
+
+	if arguments["--mqt"] == true {
+		runMessageQueueMgr(logger, routerUrl)
+	}
+
+	if arguments["--builderMgr"] == true {
+		runBuilderMgr(logger, storageSvcUrl, envBuilderNs)
+	}
+
+	if arguments["--logger"] == true {
+		runLogger()
+	}
+
+	if arguments["--storageServicePort"] != nil {
+		port := getPort(logger, arguments["--storageServicePort"])
+		filePath := arguments["--filePath"].(string)
+		runStorageSvc(logger, port, filePath)
+	}
+
+	select {}
+}

cmd/fission-cli/main.go

@@ -0,0 +1,27 @@
+/*
+Copyright 2018 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+
+	fcli "github.com/fission/fission/pkg/fission-cli"
+)
+
+func main() {
+	fcli.NewCliApp().Run(os.Args)
+}

cmd/preupgradechecks/Dockerfile.fission-preupgradechecks

@@ -0,0 +1,28 @@
+FROM golang:1.11-alpine as builder
+RUN apk add bash ca-certificates git gcc g++ libc-dev
+
+ARG GITCOMMIT=unknown
+# E.g. GITCOMMIT=$(git rev-parse HEAD)
+
+ARG BUILDVERSION=unknown
+# E.g. BUILDVERSION=$(git rev-parse HEAD)
+
+ARG BUILDDATE=unknown
+# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+
+ARG GOPKG=github.com/fission/fission
+COPY . /go/src/${GOPKG}
+WORKDIR /go/src/${GOPKG}/cmd/preupgradechecks
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a \
+    -o /go/bin/pre-upgrade-checks \
+    -gcflags=-trimpath=$GOPATH \
+    -asmflags=-trimpath=$GOPATH \
+    -ldflags "-X github.com/fission/fission/pkg/info.GitCommit=${GITCOMMIT} -X github.com/fission/fission/pkg/info.BuildDate=${BUILDDATE} -X github.com/fission/fission/pkg/info.Version=${BUILDVERSION}"
+
+
+FROM alpine:3.4
+RUN apk add --update ca-certificates
+COPY --from=builder /go/bin/pre-upgrade-checks /
+
+ENTRYPOINT ["/pre-upgrade-checks"]
+EXPOSE 8001

cmd/preupgradechecks/main.go

@@ -0,0 +1,72 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"log"
+
+	"github.com/docopt/docopt-go"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/info"
+)
+
+func getStringArgWithDefault(arg interface{}, defaultValue string) string {
+	if arg != nil {
+		return arg.(string)
+	} else {
+		return defaultValue
+	}
+}
+
+func main() {
+	logger, err := zap.NewProduction()
+	if err != nil {
+		log.Fatalf("can't initialize zap logger: %v", err)
+	}
+	defer logger.Sync()
+
+	usage := `Package to perform operations needed prior to fission installation
+Usage:
+  pre-upgrade-checks --fn-pod-namespace=<podNamespace> --envbuilder-namespace=<envBuilderNamespace>
+Options:
+  --fn-pod-namespace=<podNamespace>                        Namespace where function pods get deployed.
+  --envbuilder-namespace=<envBuilderNamespace>             Namespace where builder env pods are deployed.`
+
+	arguments, err := docopt.Parse(usage, nil, true, info.BuildInfo().String(), false)
+	if err != nil {
+		logger.Fatal("Could not parse command line arguments", zap.Error(err))
+	}
+
+	functionPodNs := getStringArgWithDefault(arguments["--fn-pod-namespace"], "fission-function")
+	envBuilderNs := getStringArgWithDefault(arguments["--envbuilder-namespace"], "fission-builder")
+
+	crdBackedClient, err := makePreUpgradeTaskClient(logger, functionPodNs, envBuilderNs)
+	if err != nil {
+		logger.Fatal("error creating a crd client, please retry helm upgrade",
+			zap.Error(err))
+	}
+
+	if !crdBackedClient.IsFissionReInstall() {
+		logger.Info("nothing to do since CRDs are not present on the cluster")
+		return
+	}
+
+	crdBackedClient.VerifyFunctionSpecReferences()
+	crdBackedClient.RemoveClusterAdminRolesForFissionSAs()
+	crdBackedClient.SetupRoleBindings()
+}

cmd/preupgradechecks/preupgradechecks.go

@@ -0,0 +1,222 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+
+	multierror "github.com/hashicorp/go-multierror"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
+	"github.com/fission/fission/pkg/crd"
+	"github.com/fission/fission/pkg/types"
+	"github.com/fission/fission/pkg/utils"
+)
+
+type (
+	PreUpgradeTaskClient struct {
+		logger        *zap.Logger
+		fissionClient *crd.FissionClient
+		k8sClient     *kubernetes.Clientset
+		apiExtClient  *apiextensionsclient.Clientset
+		fnPodNs       string
+		envBuilderNs  string
+	}
+)
+
+const (
+	maxRetries  = 5
+	FunctionCRD = "functions.fission.io"
+)
+
+func makePreUpgradeTaskClient(logger *zap.Logger, fnPodNs, envBuilderNs string) (*PreUpgradeTaskClient, error) {
+	fissionClient, k8sClient, apiExtClient, err := crd.MakeFissionClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "error making fission client")
+	}
+
+	return &PreUpgradeTaskClient{
+		logger:        logger.Named("pre_upgrade_task_client"),
+		fissionClient: fissionClient,
+		k8sClient:     k8sClient,
+		fnPodNs:       fnPodNs,
+		envBuilderNs:  envBuilderNs,
+		apiExtClient:  apiExtClient,
+	}, nil
+}
+
+// IsFissionReInstall checks if there is atleast one fission CRD, i.e. function in this case, on this cluster.
+// We need this to find out if fission had been previously installed on this cluster
+func (client *PreUpgradeTaskClient) IsFissionReInstall() bool {
+	for i := 0; i < maxRetries; i++ {
+		_, err := client.apiExtClient.ApiextensionsV1beta1().CustomResourceDefinitions().Get(FunctionCRD, metav1.GetOptions{})
+		if err != nil && k8serrors.IsNotFound(err) {
+			return false
+		}
+		if err == nil {
+			return true
+		}
+	}
+
+	return false
+}
+
+// VerifyFunctionSpecReferences verifies that a function references secrets, configmaps, pkgs in its own namespace and
+// outputs a list of functions that don't adhere to this requirement.
+func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() {
+	client.logger.Info("verifying function spec references for all functions in the cluster")
+
+	var result *multierror.Error
+	var err error
+	var fList *fv1.FunctionList
+
+	for i := 0; i < maxRetries; i++ {
+		fList, err = client.fissionClient.Functions(metav1.NamespaceAll).List(metav1.ListOptions{})
+		if err == nil {
+			break
+		}
+	}
+
+	if err != nil {
+		client.logger.Fatal("error listing functions after max retries",
+			zap.Error(err),
+			zap.Int("max_retries", maxRetries))
+	}
+
+	// check that all secrets, configmaps, packages are in the same namespace
+	for _, fn := range fList.Items {
+		secrets := fn.Spec.Secrets
+		for _, secret := range secrets {
+			if secret.Namespace != fn.Metadata.Namespace {
+				result = multierror.Append(result, fmt.Errorf("function : %s.%s cannot reference a secret : %s in namespace : %s", fn.Metadata.Name, fn.Metadata.Namespace, secret.Name, secret.Namespace))
+			}
+		}
+
+		configmaps := fn.Spec.ConfigMaps
+		for _, configmap := range configmaps {
+			if configmap.Namespace != fn.Metadata.Namespace {
+				result = multierror.Append(result, fmt.Errorf("function : %s.%s cannot reference a configmap : %s in namespace : %s", fn.Metadata.Name, fn.Metadata.Namespace, configmap.Name, configmap.Namespace))
+			}
+		}
+
+		if fn.Spec.Package.PackageRef.Namespace != fn.Metadata.Namespace {
+			result = multierror.Append(result, fmt.Errorf("function : %s.%s cannot reference a package : %s in namespace : %s", fn.Metadata.Name, fn.Metadata.Namespace, fn.Spec.Package.PackageRef.Name, fn.Spec.Package.PackageRef.Namespace))
+		}
+	}
+
+	if result != nil {
+		client.logger.Fatal("installation failed",
+			zap.Error(err),
+			zap.String("summary", "a function cannot reference secrets, configmaps and packages outside it's own namespace"))
+	}
+
+	client.logger.Info("function spec references verified")
+}
+
+// deleteClusterRoleBinding deletes the clusterRoleBinding passed as an argument to it.
+// If its not present, it just ignores and returns no errors
+func (client *PreUpgradeTaskClient) deleteClusterRoleBinding(clusterRoleBinding string) (err error) {
+	for i := 0; i < maxRetries; i++ {
+		err = client.k8sClient.RbacV1beta1().ClusterRoleBindings().Delete(clusterRoleBinding, &metav1.DeleteOptions{})
+		if err != nil && k8serrors.IsNotFound(err) || err == nil {
+			return nil
+		}
+	}
+
+	return err
+}
+
+// RemoveClusterAdminRolesForFissionSAs deletes the clusterRoleBindings previously created on this cluster
+func (client *PreUpgradeTaskClient) RemoveClusterAdminRolesForFissionSAs() {
+	clusterRoleBindings := []string{"fission-builder-crd", "fission-fetcher-crd"}
+	for _, clusterRoleBinding := range clusterRoleBindings {
+		err := client.deleteClusterRoleBinding(clusterRoleBinding)
+		if err != nil {
+			client.logger.Fatal("error deleting rolebinding",
+				zap.Error(err),
+				zap.String("role_binding", clusterRoleBinding))
+		}
+	}
+
+	client.logger.Info("femoved cluster admin privileges for fission-builder and fission-fetcher service accounts")
+}
+
+// NeedRoleBindings checks if there is atleast one package or function in default namespace.
+// It is needed to find out if package-getter-rb and secret-configmap-getter-rb needs to be created for fission-fetcher
+// and fission-builder service accounts.
+// This is because, we just deleted the ClusterRoleBindings for these service accounts in the previous function and
+// for the existing functions to work, we need to give these SAs the right privileges
+func (client *PreUpgradeTaskClient) NeedRoleBindings() bool {
+	pkgList, err := client.fissionClient.Packages(metav1.NamespaceDefault).List(metav1.ListOptions{})
+	if err == nil && len(pkgList.Items) > 0 {
+		return true
+	}
+
+	fnList, err := client.fissionClient.Functions(metav1.NamespaceDefault).List(metav1.ListOptions{})
+	if err == nil && len(fnList.Items) > 0 {
+		return true
+	}
+
+	return false
+}
+
+// Setup appropriate role bindings for fission-fetcher and fission-builder SAs
+func (client *PreUpgradeTaskClient) SetupRoleBindings() {
+	if !client.NeedRoleBindings() {
+		client.logger.Info("no fission objects found, so no role-bindings to create")
+		return
+	}
+
+	// the fact that we're here implies that there had been a prior installation of fission and objects are present still
+	// so, we go ahead and create the role-bindings necessary for the fission-fetcher and fission-builder Service Accounts.
+	err := utils.SetupRoleBinding(client.logger, client.k8sClient, types.PackageGetterRB, metav1.NamespaceDefault, types.PackageGetterCR, types.ClusterRole, types.FissionFetcherSA, client.fnPodNs)
+	if err != nil {
+		client.logger.Fatal("error setting up rolebinding for service account",
+			zap.Error(err),
+			zap.String("role_binding", types.PackageGetterRB),
+			zap.String("service_account", types.FissionFetcherSA),
+			zap.String("service_account_namespace", client.fnPodNs))
+	}
+
+	err = utils.SetupRoleBinding(client.logger, client.k8sClient, types.PackageGetterRB, metav1.NamespaceDefault, types.PackageGetterCR, types.ClusterRole, types.FissionBuilderSA, client.envBuilderNs)
+	if err != nil {
+		client.logger.Fatal("error setting up rolebinding for service account",
+			zap.Error(err),
+			zap.String("role_binding", types.PackageGetterRB),
+			zap.String("service_account", types.FissionBuilderSA),
+			zap.String("service_account_namespace", client.envBuilderNs))
+	}
+
+	err = utils.SetupRoleBinding(client.logger, client.k8sClient, types.SecretConfigMapGetterRB, metav1.NamespaceDefault, types.SecretConfigMapGetterCR, types.ClusterRole, types.FissionFetcherSA, client.fnPodNs)
+	if err != nil {
+		client.logger.Fatal("error setting up rolebinding for service account",
+			zap.Error(err),
+			zap.String("role_binding", types.SecretConfigMapGetterRB),
+			zap.String("service_account", types.FissionFetcherSA),
+			zap.String("service_account_namespace", client.fnPodNs))
+	}
+
+	client.logger.Info("created rolebindings in default namespace",
+		zap.Strings("role_bindings", []string{types.PackageGetterRB, types.SecretConfigMapGetterRB}))
+	return
+}