Upgrade tool for 0.1 -> 0.2.1 (#320)

Allows dumping v0.1 state to a json file and restoring it into a new v0.2.1 fission installation.

Usage guide is at: /Documentation/docs-site/content/upgrade-from-v0.1.md

Documentation/docs-site/content/install.md

@@ -109,7 +109,7 @@ $ helm install --namespace fission https://github.com/fission/fission/releases/d
 
 ### Install the Fission CLI
 
-#### Mac OS
+#### OS X
 
 Get the CLI binary for Mac:
 

Documentation/docs-site/content/upgrade-from-v0.1.md

@@ -34,47 +34,73 @@ rigorous about it from the beta release onwards.
 
 ## How to Upgrade
 
-1. Get the v0.2 CLI
-1. Get your Fission state from your old install
-1. Install v0.2.1
-1. Restore your Fission state
+1. Get the v0.2.1 CLI
+1. Get the Fission state from your old install
+1. Install Fission v0.2.1
+1. Restore Fission state into your new install
 1. Destroy your old install
 
 ### Get the new CLI
 
+#### OS X
+
 ```
-curl -Lo fission https://...
+$ curl -Lo fission https://github.com/fission/fission/releases/download/v0.2.1-rc/fission-cli-osx && chmod +x fission && sudo mv fission /usr/local/bin/
 ```
 
+#### Linux
+
+```
+$ curl -Lo fission https://github.com/fission/fission/releases/download/v0.2.1-rc/fission-cli-linux && chmod +x fission && sudo mv fission /usr/local/bin/
+```
+
+#### Windows
+
+For Windows, you can use the linux binary on WSL. Or you can download
+this windows executable: [fission.exe](https://github.com/fission/fission/releases/download/v0.2.1-rc/fission-cli-windows.exe)
+
 ### Get Fission state from v0.1 install
 
 ```
-fission upgrade dump
+fission --server <your V1 server> upgrade dump --file state.json
 ```
 
-This will create a JSON file with all your fission state in the current directory.
+You can skip the --server argument if you have the environment
+variable `$FISSION_URL` set to point at a v0.1 Fission server.
+
+This will create a JSON file with all your fission state in the
+current directory.
 
 ### Install the new version
 
-Follow the [install guide](../install), but you will need to ensure your two installs don't
-conflict. To do that, use separate namespaces and ensure nodeports don't conflict.  Install with a
-command similar to this:
+Read the [install guide](../install).  You can follow all of it, except that you will need to
+ensure your two installs don't conflict.  To do that, use separate namespaces and ensure nodeports
+don't conflict.  Install with a command similar to this:
 
 ```
 helm install fission-all --namespace fission2 --set controllerPort=31303,routerPort=31304,natsStreamingPort=31305,functionNamespace=fission2-function
 ```
 
+This installs fission in the `fission2` namespace and runs functions
+in the `fission2-function` namespace.
+
 ### Restore your Fission state into Fission v0.2.1
 
 ```
-fission upgrade restore
+fission upgrade restore --file state.json
 ```
 
-This uses the file created in the first step.
+This commands needs $FISSION_URL set to point to new fission installation.
+
+It uses the file created in the first step.  It doesn't modify state.json.
+
+(Note that you can run this restore on any cluster; it doesn't have the be the same kubernetes
+cluster as your old install.)
 
 ### Verify
 
-How exactly you do this is up to you! But, verify that your new install is working.
+How exactly you do this is up to you! But, at a minimum, run `fission
+fn list` to check that all the functions you expect are there.
 
 ### Switch over
 
@@ -82,7 +108,8 @@ If you had exposed fission's router to the outside world, switch over to using t
    
 ### Destroy your old install
 
-Once you're no longer using the old install, you can destroy it with:
+Once you're no longer using the old install, you can destroy it by
+deleting the namespaces that was installed in.
 
 ```
 kubectl delete namespace fission fission-function

fission/main.go

@@ -122,6 +122,11 @@ func main() {
 		{Name: "list", Usage: "List all watches", Flags: []cli.Flag{}, Action: wList},
 	}
 
+	upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"}
+	upgradeSubCommands := []cli.Command{
+		{Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState},
+		{Name: "restore", Usage: "Restore state dumped from a v0.1 install into a v0.2 install", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeRestoreState},
+	}
 	app.Commands = []cli.Command{
 		{Name: "function", Aliases: []string{"fn"}, Usage: "Create, update and manage functions", Subcommands: fnSubcommands},
 		{Name: "httptrigger", Aliases: []string{"ht", "route"}, Usage: "Manage HTTP triggers (routes) for functions", Subcommands: htSubcommands},
@@ -129,6 +134,7 @@ func main() {
 		{Name: "mqtrigger", Aliases: []string{"mqt", "messagequeue"}, Usage: "Manage message queue triggers for functions", Subcommands: mqtSubcommands},
 		{Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands},
 		{Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands},
+		{Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands},
 	}
 
 	app.Run(os.Args)

fission/upgrade.go

@@ -0,0 +1,397 @@
+package main
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/dchest/uniuri"
+	"github.com/urfave/cli"
+	"k8s.io/client-go/1.5/pkg/api"
+
+	"github.com/fission/fission"
+	"github.com/fission/fission/tpr"
+	"github.com/fission/fission/v1"
+)
+
+type (
+	V1FissionState struct {
+		Functions    []v1.Function            `json:"functions"`
+		Environments []v1.Environment         `json:"environments"`
+		Httptriggers []v1.HTTPTrigger         `json:"httptriggers"`
+		Mqtriggers   []v1.MessageQueueTrigger `json:"mqtriggers"`
+		Timetriggers []v1.TimeTrigger         `json:"timetriggers"`
+		Watches      []v1.Watch               `json:"watches"`
+		NameChanges  map[string]string        `json:"namechanges"`
+	}
+	nameRemapper struct {
+		oldToNew map[string]string
+		newNames map[string]bool
+	}
+)
+
+func getV1URL(serverUrl string) string {
+	if len(serverUrl) == 0 {
+		fatal("Need --server or FISSION_URL set to your fission server.")
+	}
+	isHTTPS := strings.Index(serverUrl, "https://") == 0
+	isHTTP := strings.Index(serverUrl, "http://") == 0
+	if !(isHTTP || isHTTPS) {
+		serverUrl = "http://" + serverUrl
+	}
+	v1url := strings.TrimSuffix(serverUrl, "/") + "/v1"
+	return v1url
+}
+
+func get(url string) []byte {
+	resp, err := http.Get(url)
+	checkErr(err, "get fission v0.1 state")
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	checkErr(err, "reading server response")
+
+	if resp.StatusCode != 200 {
+		fatal(fmt.Sprintf("Failed to fetch fission v0.1 state: %v", string(body)))
+	}
+	return body
+}
+
+// track a name in the remapper, creating a new name if needed
+func (nr *nameRemapper) trackName(old string) {
+	// all kubernetes names must match this regex
+	kubeNameRegex := "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"
+	maxLen := 63
+
+	ok, err := regexp.MatchString(kubeNameRegex, old)
+	checkErr(err, "match name regexp")
+	if ok && len(old) < maxLen {
+		// no rename
+		nr.oldToNew[old] = old
+		nr.newNames[old] = true
+		return
+	}
+
+	newName := strings.ToLower(old)
+
+	// remove disallowed
+	inv, err := regexp.Compile("[^-a-z0-9]")
+	checkErr(err, "compile regexp")
+	newName = string(inv.ReplaceAll([]byte(newName), []byte("-")))
+
+	// trim leading non-alphabetic
+	leadingnonalpha, err := regexp.Compile("^[^a-z]+")
+	checkErr(err, "compile regexp")
+	newName = string(leadingnonalpha.ReplaceAll([]byte(newName), []byte{}))
+
+	// trim trailing
+	trailing, err := regexp.Compile("[^a-z0-9]+$")
+	checkErr(err, "compile regexp")
+	newName = string(trailing.ReplaceAll([]byte(newName), []byte{}))
+
+	// truncate to length
+	if len(newName) > maxLen-4 {
+		newName = newName[0:(maxLen - 4)]
+	}
+
+	// uniqueness
+	n := newName
+	i := 0
+	for {
+		_, exists := nr.newNames[n]
+		if !exists {
+			break
+		} else {
+			i++
+			n = fmt.Sprintf("%v-%v", newName, i)
+		}
+	}
+	newName = n
+
+	// track
+	nr.oldToNew[old] = newName
+	nr.newNames[newName] = true
+}
+
+func upgradeDumpV1State(v1url string, filename string) {
+	var v1state V1FissionState
+
+	fmt.Println("Getting environments")
+	resp := get(v1url + "/environments")
+	err := json.Unmarshal(resp, &v1state.Environments)
+	checkErr(err, "parse server response")
+
+	fmt.Println("Getting watches")
+	resp = get(v1url + "/watches")
+	err = json.Unmarshal(resp, &v1state.Watches)
+	checkErr(err, "parse server response")
+
+	fmt.Println("Getting routes")
+	resp = get(v1url + "/triggers/http")
+	err = json.Unmarshal(resp, &v1state.Httptriggers)
+	checkErr(err, "parse server response")
+
+	fmt.Println("Getting message queue triggers")
+	resp = get(v1url + "/triggers/messagequeue")
+	err = json.Unmarshal(resp, &v1state.Mqtriggers)
+	checkErr(err, "parse server response")
+
+	fmt.Println("Getting time triggers")
+	resp = get(v1url + "/triggers/time")
+	err = json.Unmarshal(resp, &v1state.Timetriggers)
+	checkErr(err, "parse server response")
+
+	fmt.Println("Getting function list")
+	resp = get(v1url + "/functions")
+	err = json.Unmarshal(resp, &v1state.Functions)
+	checkErr(err, "parse server response")
+
+	// we have to change names that are disallowed in kubernetes
+	nr := nameRemapper{
+		oldToNew: make(map[string]string),
+		newNames: make(map[string]bool),
+	}
+
+	// get all referenced function metadata
+	funcMetaSet := make(map[v1.Metadata]bool)
+	for _, f := range v1state.Functions {
+		funcMetaSet[f.Metadata] = true
+		nr.trackName(f.Metadata.Name)
+	}
+	for _, t := range v1state.Httptriggers {
+		funcMetaSet[t.Function] = true
+		nr.trackName(t.Metadata.Name)
+	}
+	for _, t := range v1state.Mqtriggers {
+		funcMetaSet[t.Function] = true
+		nr.trackName(t.Metadata.Name)
+	}
+	for _, t := range v1state.Watches {
+		funcMetaSet[t.Function] = true
+		nr.trackName(t.Metadata.Name)
+	}
+	for _, t := range v1state.Timetriggers {
+		funcMetaSet[t.Function] = true
+		nr.trackName(t.Metadata.Name)
+	}
+
+	for _, e := range v1state.Environments {
+		nr.trackName(e.Metadata.Name)
+	}
+
+	fmt.Println("Getting functions")
+	// get each function
+	funcs := make(map[v1.Metadata]v1.Function)
+	for m := range funcMetaSet {
+		if len(m.Uid) != 0 {
+			resp = get(fmt.Sprintf("%v/functions/%v?uid=%v", v1url, m.Name, m.Uid))
+		} else {
+			resp = get(fmt.Sprintf("%v/functions/%v", v1url, m.Name))
+		}
+
+		var f v1.Function
+
+		// unmarshal
+		err = json.Unmarshal(resp, &f)
+		checkErr(err, "parse server response")
+
+		// load into a map to remove duplicates
+		funcs[f.Metadata] = f
+	}
+
+	// add list of unique functions to v1state from map
+	v1state.Functions = make([]v1.Function, 0)
+	for _, f := range funcs {
+		v1state.Functions = append(v1state.Functions, f)
+	}
+
+	// dump name changes
+	v1state.NameChanges = nr.oldToNew
+
+	// serialize v1state
+	out, err := json.MarshalIndent(v1state, "", "    ")
+	checkErr(err, "serialize v0.1 state")
+
+	// dump to file fission-v01-state.json
+	if len(filename) == 0 {
+		filename = "fission-v01-state.json"
+	}
+	err = ioutil.WriteFile(filename, out, 0644)
+	checkErr(err, "write file")
+
+	fmt.Printf("Done: Saved %v functions, %v HTTP triggers, %v watches, %v message queue triggers, %v time triggers.\n",
+		len(v1state.Functions), len(v1state.Httptriggers), len(v1state.Watches), len(v1state.Mqtriggers), len(v1state.Timetriggers))
+}
+
+func functionRefFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *fission.FunctionReference {
+	return &fission.FunctionReference{
+		Type: fission.FunctionReferenceTypeFunctionName,
+		Name: nameRemap[m.Name],
+	}
+}
+
+func tprMetadataFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *api.ObjectMeta {
+	return &api.ObjectMeta{
+		Name:      nameRemap[m.Name],
+		Namespace: api.NamespaceDefault,
+	}
+}
+
+func upgradeDumpState(c *cli.Context) error {
+	u := getV1URL(c.GlobalString("server"))
+	filename := c.String("file")
+
+	// check v1
+	resp, err := http.Get(u + "/environments")
+	checkErr(err, "reach fission server")
+	if resp.StatusCode == 404 {
+		msg := fmt.Sprintf("Server %v isn't a v1 Fission server. Use --server to point at a pre-0.2.x Fission server.", u)
+		fatal(msg)
+	}
+
+	upgradeDumpV1State(u, filename)
+	return nil
+}
+
+func upgradeRestoreState(c *cli.Context) error {
+	filename := c.String("file")
+	if len(filename) == 0 {
+		filename = "fission-v01-state.json"
+	}
+
+	contents, err := ioutil.ReadFile(filename)
+	checkErr(err, fmt.Sprintf("open file %v", filename))
+
+	var v1state V1FissionState
+	err = json.Unmarshal(contents, &v1state)
+	checkErr(err, "parse dumped v1 state")
+
+	// create a regular v2 client
+	client := getClient(c.GlobalString("server"))
+
+	// create functions
+	for _, f := range v1state.Functions {
+
+		// get post-rename function name, derive pkg name from it
+		fnName := v1state.NameChanges[f.Metadata.Name]
+		pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6)))
+
+		// write function to file
+		tmpfile, err := ioutil.TempFile("", pkgName)
+		checkErr(err, "create temporary file")
+		code, err := base64.StdEncoding.DecodeString(f.Code)
+		checkErr(err, "decode base64 function contents")
+		tmpfile.Write(code)
+		tmpfile.Sync()
+		tmpfile.Close()
+
+		// upload
+		archive := createArchive(client, tmpfile.Name())
+		os.Remove(tmpfile.Name())
+
+		// create pkg
+		pkgSpec := fission.PackageSpec{
+			Environment: fission.EnvironmentReference{
+				Name:      v1state.NameChanges[f.Environment.Name],
+				Namespace: api.NamespaceDefault,
+			},
+			Deployment: *archive,
+		}
+		pkg, err := client.PackageCreate(&tpr.Package{
+			Metadata: api.ObjectMeta{
+				Name:      pkgName,
+				Namespace: api.NamespaceDefault,
+			},
+			Spec: pkgSpec,
+		})
+		checkErr(err, fmt.Sprintf("create package %v", pkgName))
+		_, err = client.FunctionCreate(&tpr.Function{
+			Metadata: *tprMetadataFromV1Metadata(&f.Metadata, v1state.NameChanges),
+			Spec: fission.FunctionSpec{
+				Environment: pkgSpec.Environment,
+				Package: fission.FunctionPackageRef{
+					PackageRef: fission.PackageRef{
+						Name:            pkg.Name,
+						Namespace:       pkg.Namespace,
+						ResourceVersion: pkg.ResourceVersion,
+					},
+				},
+			},
+		})
+		checkErr(err, fmt.Sprintf("create function %v", v1state.NameChanges[f.Metadata.Name]))
+
+	}
+
+	// create envs
+	for _, e := range v1state.Environments {
+		_, err = client.EnvironmentCreate(&tpr.Environment{
+			Metadata: *tprMetadataFromV1Metadata(&e.Metadata, v1state.NameChanges),
+			Spec: fission.EnvironmentSpec{
+				Version: 1,
+				Runtime: fission.Runtime{
+					Image: e.RunContainerImageUrl,
+				},
+			},
+		})
+		checkErr(err, fmt.Sprintf("create environment %v", e.Metadata.Name))
+	}
+
+	// create httptriggers
+	for _, t := range v1state.Httptriggers {
+		_, err = client.HTTPTriggerCreate(&tpr.Httptrigger{
+			Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
+			Spec: fission.HTTPTriggerSpec{
+				RelativeURL:       t.UrlPattern,
+				Method:            t.Method,
+				FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges),
+			},
+		})
+		checkErr(err, fmt.Sprintf("create http trigger %v", t.Metadata.Name))
+	}
+
+	// create mqtriggers
+	for _, t := range v1state.Mqtriggers {
+		_, err = client.MessageQueueTriggerCreate(&tpr.Messagequeuetrigger{
+			Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
+			Spec: fission.MessageQueueTriggerSpec{
+				FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges),
+				MessageQueueType:  t.MessageQueueType,
+				Topic:             t.Topic,
+				ResponseTopic:     t.ResponseTopic,
+			},
+		})
+		checkErr(err, fmt.Sprintf("create http trigger %v", t.Metadata.Name))
+	}
+
+	// create time triggers
+	for _, t := range v1state.Timetriggers {
+		_, err = client.TimeTriggerCreate(&tpr.Timetrigger{
+			Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
+			Spec: fission.TimeTriggerSpec{
+				FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges),
+				Cron:              t.Cron,
+			},
+		})
+		checkErr(err, fmt.Sprintf("create time trigger %v", t.Metadata.Name))
+	}
+
+	// create watches
+	for _, t := range v1state.Watches {
+		_, err = client.WatchCreate(&tpr.Kuberneteswatchtrigger{
+			Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
+			Spec: fission.KubernetesWatchTriggerSpec{
+				Namespace:         t.Namespace,
+				Type:              t.ObjType,
+				FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges),
+			},
+		})
+		checkErr(err, fmt.Sprintf("create kubernetes watch trigger %v", t.Metadata.Name))
+	}
+
+	return nil
+}

v1/types.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+//
+// These are types from the v1 API, and are only preserved for
+// compatibility.  They should never be changed.
+//
+
+type (
+	// Metadata is used as the general identifier for all kinds of
+	// resources managed by the controller.
+	Metadata struct {
+		Name string `json:"name"`
+		Uid  string `json:"uid,omitempty"`
+	}
+
+	// Function is a unit of executable code.  Though it's called
+	// a function, the code may have more than one function; it's
+	// usually some sort of module or package.
+	Function struct {
+		Metadata    `json:"metadata"`
+		Environment Metadata `json:"environment"`
+		Code        string   `json:"code"`
+	}
+
+	// Environment identifies the language and OS specific
+	// resources that a function depends on.  For now this
+	// includes only the function run container image.  Later,
+	// this will also include build containers, as well as support
+	// tools like debuggers, profilers, etc.
+	Environment struct {
+		Metadata             `json:"metadata"`
+		RunContainerImageUrl string `json:"runContainerImageUrl"`
+	}
+
+	// HTTPTrigger maps URL patterns to functions.  Function.UID
+	// is optional; if absent, the latest version of the function
+	// will automatically be selected.
+	HTTPTrigger struct {
+		Metadata   `json:"metadata"`
+		UrlPattern string   `json:"urlpattern"`
+		Method     string   `json:"method"`
+		Function   Metadata `json:"function"`
+	}
+
+	MessageQueueTrigger struct {
+		Metadata         `json:"metadata"`
+		Function         Metadata `json:"function"`
+		MessageQueueType string   `json:"messageQueueType"`
+		Topic            string   `json:"topic"`
+		ResponseTopic    string   `json:"respTopic,omitempty"`
+	}
+
+	// Watch is a specification of Kubernetes watch along with a URL to post events to.
+	Watch struct {
+		Metadata `json:"metadata"`
+
+		Namespace     string `json:"namespace"`
+		ObjType       string `json:"objtype"`
+		LabelSelector string `json:"labelselector"`
+		FieldSelector string `json:"fieldselector"`
+
+		Function Metadata `json:"function"`
+
+		Target string `json:"target"` // Watch publish target (URL, NATS stream, etc)
+	}
+
+	// TimeTrigger invokes the specific function at a time or
+	// times specified by a cron string.
+	TimeTrigger struct {
+		Metadata `json:"metadata"`
+
+		Cron string `json:"cron"`
+
+		Function Metadata `json:"function"`
+	}
+
+	// Errors returned by the Fission API.
+	Error struct {
+		Code    errorCode `json:"code"`
+		Message string    `json:"message"`
+	}
+
+	errorCode int
+)
+
+const (
+	ErrorInternal = iota
+
+	ErrorNotAuthorized
+	ErrorNotFound
+	ErrorNameExists
+	ErrorInvalidArgument
+	ErrorNoSpace
+	ErrorNotImplmented
+)
+
+// must match order and len of the above const
+var errorDescriptions = []string{
+	"Internal error",
+	"Not authorized",
+	"Resource not found",
+	"Resource exists",
+	"Invalid argument",
+	"No space",
+	"Not implemented",
+}