From 4287d140002d03e3e8f9798ca1ba015966f9afd5 Mon Sep 17 00:00:00 2001 From: Soam Vasani Date: Mon, 11 Sep 2017 18:31:36 -0700 Subject: [PATCH] Upgrade tool for 0.1 -> 0.2.1 (#320) Allows dumping v0.1 state to a json file and restoring it into a new v0.2.1 fission installation. Usage guide is at: /Documentation/docs-site/content/upgrade-from-v0.1.md --- Documentation/docs-site/content/install.md | 2 +- .../docs-site/content/upgrade-from-v0.1.md | 55 ++- fission/main.go | 6 + fission/upgrade.go | 397 ++++++++++++++++++ v1/types.go | 122 ++++++ 5 files changed, 567 insertions(+), 15 deletions(-) create mode 100644 fission/upgrade.go create mode 100644 v1/types.go diff --git a/Documentation/docs-site/content/install.md b/Documentation/docs-site/content/install.md index e1564d9c..1e5fad22 100644 --- a/Documentation/docs-site/content/install.md +++ b/Documentation/docs-site/content/install.md @@ -109,7 +109,7 @@ $ helm install --namespace fission https://github.com/fission/fission/releases/d ### Install the Fission CLI -#### Mac OS +#### OS X Get the CLI binary for Mac: diff --git a/Documentation/docs-site/content/upgrade-from-v0.1.md b/Documentation/docs-site/content/upgrade-from-v0.1.md index 725ec960..ac56cd0d 100644 --- a/Documentation/docs-site/content/upgrade-from-v0.1.md +++ b/Documentation/docs-site/content/upgrade-from-v0.1.md @@ -34,47 +34,73 @@ rigorous about it from the beta release onwards. ## How to Upgrade -1. Get the v0.2 CLI -1. Get your Fission state from your old install -1. Install v0.2.1 -1. Restore your Fission state +1. Get the v0.2.1 CLI +1. Get the Fission state from your old install +1. Install Fission v0.2.1 +1. Restore Fission state into your new install 1. Destroy your old install ### Get the new CLI +#### OS X + ``` -curl -Lo fission https://... +$ curl -Lo fission https://github.com/fission/fission/releases/download/v0.2.1-rc/fission-cli-osx && chmod +x fission && sudo mv fission /usr/local/bin/ ``` +#### Linux + +``` +$ curl -Lo fission https://github.com/fission/fission/releases/download/v0.2.1-rc/fission-cli-linux && chmod +x fission && sudo mv fission /usr/local/bin/ +``` + +#### Windows + +For Windows, you can use the linux binary on WSL. Or you can download +this windows executable: [fission.exe](https://github.com/fission/fission/releases/download/v0.2.1-rc/fission-cli-windows.exe) + ### Get Fission state from v0.1 install ``` -fission upgrade dump +fission --server upgrade dump --file state.json ``` -This will create a JSON file with all your fission state in the current directory. +You can skip the --server argument if you have the environment +variable `$FISSION_URL` set to point at a v0.1 Fission server. + +This will create a JSON file with all your fission state in the +current directory. ### Install the new version -Follow the [install guide](../install), but you will need to ensure your two installs don't -conflict. To do that, use separate namespaces and ensure nodeports don't conflict. Install with a -command similar to this: +Read the [install guide](../install). You can follow all of it, except that you will need to +ensure your two installs don't conflict. To do that, use separate namespaces and ensure nodeports +don't conflict. Install with a command similar to this: ``` helm install fission-all --namespace fission2 --set controllerPort=31303,routerPort=31304,natsStreamingPort=31305,functionNamespace=fission2-function ``` +This installs fission in the `fission2` namespace and runs functions +in the `fission2-function` namespace. + ### Restore your Fission state into Fission v0.2.1 ``` -fission upgrade restore +fission upgrade restore --file state.json ``` -This uses the file created in the first step. +This commands needs $FISSION_URL set to point to new fission installation. + +It uses the file created in the first step. It doesn't modify state.json. + +(Note that you can run this restore on any cluster; it doesn't have the be the same kubernetes +cluster as your old install.) ### Verify -How exactly you do this is up to you! But, verify that your new install is working. +How exactly you do this is up to you! But, at a minimum, run `fission +fn list` to check that all the functions you expect are there. ### Switch over @@ -82,7 +108,8 @@ If you had exposed fission's router to the outside world, switch over to using t ### Destroy your old install -Once you're no longer using the old install, you can destroy it with: +Once you're no longer using the old install, you can destroy it by +deleting the namespaces that was installed in. ``` kubectl delete namespace fission fission-function diff --git a/fission/main.go b/fission/main.go index 159ce8f0..87be32a4 100644 --- a/fission/main.go +++ b/fission/main.go @@ -122,6 +122,11 @@ func main() { {Name: "list", Usage: "List all watches", Flags: []cli.Flag{}, Action: wList}, } + upgradeFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all fission state"} + upgradeSubCommands := []cli.Command{ + {Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState}, + {Name: "restore", Usage: "Restore state dumped from a v0.1 install into a v0.2 install", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeRestoreState}, + } app.Commands = []cli.Command{ {Name: "function", Aliases: []string{"fn"}, Usage: "Create, update and manage functions", Subcommands: fnSubcommands}, {Name: "httptrigger", Aliases: []string{"ht", "route"}, Usage: "Manage HTTP triggers (routes) for functions", Subcommands: htSubcommands}, @@ -129,6 +134,7 @@ func main() { {Name: "mqtrigger", Aliases: []string{"mqt", "messagequeue"}, Usage: "Manage message queue triggers for functions", Subcommands: mqtSubcommands}, {Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands}, {Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands}, + {Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands}, } app.Run(os.Args) diff --git a/fission/upgrade.go b/fission/upgrade.go new file mode 100644 index 00000000..8e9d1dd1 --- /dev/null +++ b/fission/upgrade.go @@ -0,0 +1,397 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "regexp" + "strings" + + "github.com/dchest/uniuri" + "github.com/urfave/cli" + "k8s.io/client-go/1.5/pkg/api" + + "github.com/fission/fission" + "github.com/fission/fission/tpr" + "github.com/fission/fission/v1" +) + +type ( + V1FissionState struct { + Functions []v1.Function `json:"functions"` + Environments []v1.Environment `json:"environments"` + Httptriggers []v1.HTTPTrigger `json:"httptriggers"` + Mqtriggers []v1.MessageQueueTrigger `json:"mqtriggers"` + Timetriggers []v1.TimeTrigger `json:"timetriggers"` + Watches []v1.Watch `json:"watches"` + NameChanges map[string]string `json:"namechanges"` + } + nameRemapper struct { + oldToNew map[string]string + newNames map[string]bool + } +) + +func getV1URL(serverUrl string) string { + if len(serverUrl) == 0 { + fatal("Need --server or FISSION_URL set to your fission server.") + } + isHTTPS := strings.Index(serverUrl, "https://") == 0 + isHTTP := strings.Index(serverUrl, "http://") == 0 + if !(isHTTP || isHTTPS) { + serverUrl = "http://" + serverUrl + } + v1url := strings.TrimSuffix(serverUrl, "/") + "/v1" + return v1url +} + +func get(url string) []byte { + resp, err := http.Get(url) + checkErr(err, "get fission v0.1 state") + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + checkErr(err, "reading server response") + + if resp.StatusCode != 200 { + fatal(fmt.Sprintf("Failed to fetch fission v0.1 state: %v", string(body))) + } + return body +} + +// track a name in the remapper, creating a new name if needed +func (nr *nameRemapper) trackName(old string) { + // all kubernetes names must match this regex + kubeNameRegex := "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" + maxLen := 63 + + ok, err := regexp.MatchString(kubeNameRegex, old) + checkErr(err, "match name regexp") + if ok && len(old) < maxLen { + // no rename + nr.oldToNew[old] = old + nr.newNames[old] = true + return + } + + newName := strings.ToLower(old) + + // remove disallowed + inv, err := regexp.Compile("[^-a-z0-9]") + checkErr(err, "compile regexp") + newName = string(inv.ReplaceAll([]byte(newName), []byte("-"))) + + // trim leading non-alphabetic + leadingnonalpha, err := regexp.Compile("^[^a-z]+") + checkErr(err, "compile regexp") + newName = string(leadingnonalpha.ReplaceAll([]byte(newName), []byte{})) + + // trim trailing + trailing, err := regexp.Compile("[^a-z0-9]+$") + checkErr(err, "compile regexp") + newName = string(trailing.ReplaceAll([]byte(newName), []byte{})) + + // truncate to length + if len(newName) > maxLen-4 { + newName = newName[0:(maxLen - 4)] + } + + // uniqueness + n := newName + i := 0 + for { + _, exists := nr.newNames[n] + if !exists { + break + } else { + i++ + n = fmt.Sprintf("%v-%v", newName, i) + } + } + newName = n + + // track + nr.oldToNew[old] = newName + nr.newNames[newName] = true +} + +func upgradeDumpV1State(v1url string, filename string) { + var v1state V1FissionState + + fmt.Println("Getting environments") + resp := get(v1url + "/environments") + err := json.Unmarshal(resp, &v1state.Environments) + checkErr(err, "parse server response") + + fmt.Println("Getting watches") + resp = get(v1url + "/watches") + err = json.Unmarshal(resp, &v1state.Watches) + checkErr(err, "parse server response") + + fmt.Println("Getting routes") + resp = get(v1url + "/triggers/http") + err = json.Unmarshal(resp, &v1state.Httptriggers) + checkErr(err, "parse server response") + + fmt.Println("Getting message queue triggers") + resp = get(v1url + "/triggers/messagequeue") + err = json.Unmarshal(resp, &v1state.Mqtriggers) + checkErr(err, "parse server response") + + fmt.Println("Getting time triggers") + resp = get(v1url + "/triggers/time") + err = json.Unmarshal(resp, &v1state.Timetriggers) + checkErr(err, "parse server response") + + fmt.Println("Getting function list") + resp = get(v1url + "/functions") + err = json.Unmarshal(resp, &v1state.Functions) + checkErr(err, "parse server response") + + // we have to change names that are disallowed in kubernetes + nr := nameRemapper{ + oldToNew: make(map[string]string), + newNames: make(map[string]bool), + } + + // get all referenced function metadata + funcMetaSet := make(map[v1.Metadata]bool) + for _, f := range v1state.Functions { + funcMetaSet[f.Metadata] = true + nr.trackName(f.Metadata.Name) + } + for _, t := range v1state.Httptriggers { + funcMetaSet[t.Function] = true + nr.trackName(t.Metadata.Name) + } + for _, t := range v1state.Mqtriggers { + funcMetaSet[t.Function] = true + nr.trackName(t.Metadata.Name) + } + for _, t := range v1state.Watches { + funcMetaSet[t.Function] = true + nr.trackName(t.Metadata.Name) + } + for _, t := range v1state.Timetriggers { + funcMetaSet[t.Function] = true + nr.trackName(t.Metadata.Name) + } + + for _, e := range v1state.Environments { + nr.trackName(e.Metadata.Name) + } + + fmt.Println("Getting functions") + // get each function + funcs := make(map[v1.Metadata]v1.Function) + for m := range funcMetaSet { + if len(m.Uid) != 0 { + resp = get(fmt.Sprintf("%v/functions/%v?uid=%v", v1url, m.Name, m.Uid)) + } else { + resp = get(fmt.Sprintf("%v/functions/%v", v1url, m.Name)) + } + + var f v1.Function + + // unmarshal + err = json.Unmarshal(resp, &f) + checkErr(err, "parse server response") + + // load into a map to remove duplicates + funcs[f.Metadata] = f + } + + // add list of unique functions to v1state from map + v1state.Functions = make([]v1.Function, 0) + for _, f := range funcs { + v1state.Functions = append(v1state.Functions, f) + } + + // dump name changes + v1state.NameChanges = nr.oldToNew + + // serialize v1state + out, err := json.MarshalIndent(v1state, "", " ") + checkErr(err, "serialize v0.1 state") + + // dump to file fission-v01-state.json + if len(filename) == 0 { + filename = "fission-v01-state.json" + } + err = ioutil.WriteFile(filename, out, 0644) + checkErr(err, "write file") + + fmt.Printf("Done: Saved %v functions, %v HTTP triggers, %v watches, %v message queue triggers, %v time triggers.\n", + len(v1state.Functions), len(v1state.Httptriggers), len(v1state.Watches), len(v1state.Mqtriggers), len(v1state.Timetriggers)) +} + +func functionRefFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *fission.FunctionReference { + return &fission.FunctionReference{ + Type: fission.FunctionReferenceTypeFunctionName, + Name: nameRemap[m.Name], + } +} + +func tprMetadataFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *api.ObjectMeta { + return &api.ObjectMeta{ + Name: nameRemap[m.Name], + Namespace: api.NamespaceDefault, + } +} + +func upgradeDumpState(c *cli.Context) error { + u := getV1URL(c.GlobalString("server")) + filename := c.String("file") + + // check v1 + resp, err := http.Get(u + "/environments") + checkErr(err, "reach fission server") + if resp.StatusCode == 404 { + msg := fmt.Sprintf("Server %v isn't a v1 Fission server. Use --server to point at a pre-0.2.x Fission server.", u) + fatal(msg) + } + + upgradeDumpV1State(u, filename) + return nil +} + +func upgradeRestoreState(c *cli.Context) error { + filename := c.String("file") + if len(filename) == 0 { + filename = "fission-v01-state.json" + } + + contents, err := ioutil.ReadFile(filename) + checkErr(err, fmt.Sprintf("open file %v", filename)) + + var v1state V1FissionState + err = json.Unmarshal(contents, &v1state) + checkErr(err, "parse dumped v1 state") + + // create a regular v2 client + client := getClient(c.GlobalString("server")) + + // create functions + for _, f := range v1state.Functions { + + // get post-rename function name, derive pkg name from it + fnName := v1state.NameChanges[f.Metadata.Name] + pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6))) + + // write function to file + tmpfile, err := ioutil.TempFile("", pkgName) + checkErr(err, "create temporary file") + code, err := base64.StdEncoding.DecodeString(f.Code) + checkErr(err, "decode base64 function contents") + tmpfile.Write(code) + tmpfile.Sync() + tmpfile.Close() + + // upload + archive := createArchive(client, tmpfile.Name()) + os.Remove(tmpfile.Name()) + + // create pkg + pkgSpec := fission.PackageSpec{ + Environment: fission.EnvironmentReference{ + Name: v1state.NameChanges[f.Environment.Name], + Namespace: api.NamespaceDefault, + }, + Deployment: *archive, + } + pkg, err := client.PackageCreate(&tpr.Package{ + Metadata: api.ObjectMeta{ + Name: pkgName, + Namespace: api.NamespaceDefault, + }, + Spec: pkgSpec, + }) + checkErr(err, fmt.Sprintf("create package %v", pkgName)) + _, err = client.FunctionCreate(&tpr.Function{ + Metadata: *tprMetadataFromV1Metadata(&f.Metadata, v1state.NameChanges), + Spec: fission.FunctionSpec{ + Environment: pkgSpec.Environment, + Package: fission.FunctionPackageRef{ + PackageRef: fission.PackageRef{ + Name: pkg.Name, + Namespace: pkg.Namespace, + ResourceVersion: pkg.ResourceVersion, + }, + }, + }, + }) + checkErr(err, fmt.Sprintf("create function %v", v1state.NameChanges[f.Metadata.Name])) + + } + + // create envs + for _, e := range v1state.Environments { + _, err = client.EnvironmentCreate(&tpr.Environment{ + Metadata: *tprMetadataFromV1Metadata(&e.Metadata, v1state.NameChanges), + Spec: fission.EnvironmentSpec{ + Version: 1, + Runtime: fission.Runtime{ + Image: e.RunContainerImageUrl, + }, + }, + }) + checkErr(err, fmt.Sprintf("create environment %v", e.Metadata.Name)) + } + + // create httptriggers + for _, t := range v1state.Httptriggers { + _, err = client.HTTPTriggerCreate(&tpr.Httptrigger{ + Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges), + Spec: fission.HTTPTriggerSpec{ + RelativeURL: t.UrlPattern, + Method: t.Method, + FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges), + }, + }) + checkErr(err, fmt.Sprintf("create http trigger %v", t.Metadata.Name)) + } + + // create mqtriggers + for _, t := range v1state.Mqtriggers { + _, err = client.MessageQueueTriggerCreate(&tpr.Messagequeuetrigger{ + Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges), + Spec: fission.MessageQueueTriggerSpec{ + FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges), + MessageQueueType: t.MessageQueueType, + Topic: t.Topic, + ResponseTopic: t.ResponseTopic, + }, + }) + checkErr(err, fmt.Sprintf("create http trigger %v", t.Metadata.Name)) + } + + // create time triggers + for _, t := range v1state.Timetriggers { + _, err = client.TimeTriggerCreate(&tpr.Timetrigger{ + Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges), + Spec: fission.TimeTriggerSpec{ + FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges), + Cron: t.Cron, + }, + }) + checkErr(err, fmt.Sprintf("create time trigger %v", t.Metadata.Name)) + } + + // create watches + for _, t := range v1state.Watches { + _, err = client.WatchCreate(&tpr.Kuberneteswatchtrigger{ + Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges), + Spec: fission.KubernetesWatchTriggerSpec{ + Namespace: t.Namespace, + Type: t.ObjType, + FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges), + }, + }) + checkErr(err, fmt.Sprintf("create kubernetes watch trigger %v", t.Metadata.Name)) + } + + return nil +} diff --git a/v1/types.go b/v1/types.go new file mode 100644 index 00000000..8e000cb2 --- /dev/null +++ b/v1/types.go @@ -0,0 +1,122 @@ +/* +Copyright 2016 The Fission Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +// +// These are types from the v1 API, and are only preserved for +// compatibility. They should never be changed. +// + +type ( + // Metadata is used as the general identifier for all kinds of + // resources managed by the controller. + Metadata struct { + Name string `json:"name"` + Uid string `json:"uid,omitempty"` + } + + // Function is a unit of executable code. Though it's called + // a function, the code may have more than one function; it's + // usually some sort of module or package. + Function struct { + Metadata `json:"metadata"` + Environment Metadata `json:"environment"` + Code string `json:"code"` + } + + // Environment identifies the language and OS specific + // resources that a function depends on. For now this + // includes only the function run container image. Later, + // this will also include build containers, as well as support + // tools like debuggers, profilers, etc. + Environment struct { + Metadata `json:"metadata"` + RunContainerImageUrl string `json:"runContainerImageUrl"` + } + + // HTTPTrigger maps URL patterns to functions. Function.UID + // is optional; if absent, the latest version of the function + // will automatically be selected. + HTTPTrigger struct { + Metadata `json:"metadata"` + UrlPattern string `json:"urlpattern"` + Method string `json:"method"` + Function Metadata `json:"function"` + } + + MessageQueueTrigger struct { + Metadata `json:"metadata"` + Function Metadata `json:"function"` + MessageQueueType string `json:"messageQueueType"` + Topic string `json:"topic"` + ResponseTopic string `json:"respTopic,omitempty"` + } + + // Watch is a specification of Kubernetes watch along with a URL to post events to. + Watch struct { + Metadata `json:"metadata"` + + Namespace string `json:"namespace"` + ObjType string `json:"objtype"` + LabelSelector string `json:"labelselector"` + FieldSelector string `json:"fieldselector"` + + Function Metadata `json:"function"` + + Target string `json:"target"` // Watch publish target (URL, NATS stream, etc) + } + + // TimeTrigger invokes the specific function at a time or + // times specified by a cron string. + TimeTrigger struct { + Metadata `json:"metadata"` + + Cron string `json:"cron"` + + Function Metadata `json:"function"` + } + + // Errors returned by the Fission API. + Error struct { + Code errorCode `json:"code"` + Message string `json:"message"` + } + + errorCode int +) + +const ( + ErrorInternal = iota + + ErrorNotAuthorized + ErrorNotFound + ErrorNameExists + ErrorInvalidArgument + ErrorNoSpace + ErrorNotImplmented +) + +// must match order and len of the above const +var errorDescriptions = []string{ + "Internal error", + "Not authorized", + "Resource not found", + "Resource exists", + "Invalid argument", + "No space", + "Not implemented", +}