Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch from ThirdPartyResources to CustomResourceDefinitions (#381)

Browse Source 
Switch Fission's storage over to the new CustomResourceDefinitions, from the deprecated ThirdPartyResources. This allows us to be compatible with Kubernets 1.8 and onwards.

This also adds a CLI tool for dumping state from an old fission version and restoring state into new CRDs.

The storage service is unaffected by this change.
This commit is contained in:
Ta-Ching Chen
2017-11-04 14:55:16 -07:00
committed by Soam Vasani
parent 746c51901d
commit 5f14b9b0ae
76 changed files with 1223 additions and 701 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fission/environment.go
+2 -2
View File
@@ -25,7 +25,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/tpr"
"github.com/fission/fission/crd"
)
func envCreate(c *cli.Context) error {
@@ -59,7 +59,7 @@ func envCreate(c *cli.Context) error {
envVersion = 1
}
env := &tpr.Environment{
env := &crd.Environment{
Metadata: metav1.ObjectMeta{
Name: envName,
Namespace: metav1.NamespaceDefault,
fission/function.go
+4 -4
View File
@@ -37,9 +37,9 @@ import (
"github.com/fission/fission"
"github.com/fission/fission/controller/client"
"github.com/fission/fission/crd"
"github.com/fission/fission/fission/logdb"
storageSvcClient "github.com/fission/fission/storagesvc/client"
"github.com/fission/fission/tpr"
)
func fileSize(filePath string) int64 {
@@ -113,7 +113,7 @@ func createPackage(client *client.Client, envName, srcArchiveName, deployArchive
}
pkgName := strings.ToLower(uuid.NewV4().String())
pkg := &tpr.Package{
pkg := &crd.Package{
Metadata: metav1.ObjectMeta{
Name: pkgName,
Namespace: metav1.NamespaceDefault,
@@ -200,7 +200,7 @@ func fnCreate(c *cli.Context) error {
pkgMetadata := createPackage(client, envName, srcArchiveName, deployArchiveName, buildcmd)
function := &tpr.Function{
function := &crd.Function{
Metadata: metav1.ObjectMeta{
Name: fnName,
Namespace: metav1.NamespaceDefault,
@@ -236,7 +236,7 @@ func fnCreate(c *cli.Context) error {
method = "GET"
}
triggerName := uuid.NewV4().String()
ht := &tpr.Httptrigger{
ht := &crd.HTTPTrigger{
Metadata: metav1.ObjectMeta{
Name: triggerName,
Namespace: metav1.NamespaceDefault,
fission/httptrigger.go
+2 -2
View File
@@ -28,7 +28,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/tpr"
"github.com/fission/fission/crd"
)
// returns one of http.Method*
@@ -76,7 +76,7 @@ func htCreate(c *cli.Context) error {
// just name triggers by uuid.
triggerName := uuid.NewV4().String()
ht := &tpr.Httptrigger{
ht := &crd.HTTPTrigger{
Metadata: metav1.ObjectMeta{
Name: triggerName,
Namespace: metav1.NamespaceDefault,
fission/main.go
+9
View File
@@ -135,6 +135,14 @@ func main() {
{Name: "dump", Usage: "Dump all state from a v0.1 fission installation", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeDumpState},
{Name: "restore", Usage: "Restore state dumped from a v0.1 install into a v0.2+ install", Flags: []cli.Flag{upgradeFileFlag}, Action: upgradeRestoreState},
}
migrateFileFlag := cli.StringFlag{Name: "file", Usage: "JSON file containing all CRDs"}
migrateSubCommands := []cli.Command{
{Name: "dump", Usage: "Dump all state from a pre-0.4 Fission installation (which used ThirdPartyResources) into a JSON file", Flags: []cli.Flag{migrateFileFlag}, Action: migrateDumpTPR},
{Name: "delete", Usage: "Delete all TPRs", Flags: []cli.Flag{}, Action: migrateDeleteTPR},
{Name: "restore", Usage: "Restore state dumped from a pre-0.4 Fission cluster. Requires Fission 0.4, which uses Kubernetes CustomResources.", Flags: []cli.Flag{migrateFileFlag}, Action: migrateRestoreCRD},
}
app.Commands = []cli.Command{
{Name: "function", Aliases: []string{"fn"}, Usage: "Create, update and manage functions", Subcommands: fnSubcommands},
{Name: "httptrigger", Aliases: []string{"ht", "route"}, Usage: "Manage HTTP triggers (routes) for functions", Subcommands: htSubcommands},
@@ -143,6 +151,7 @@ func main() {
{Name: "environment", Aliases: []string{"env"}, Usage: "Manage environments", Subcommands: envSubcommands},
{Name: "watch", Aliases: []string{"w"}, Usage: "Manage watches", Subcommands: wSubCommands},
{Name: "upgrade", Aliases: []string{}, Usage: "Upgrade tool from fission v0.1", Subcommands: upgradeSubCommands},
{Name: "tpr2crd", Aliases: []string{}, Usage: "Migrate tool for TPR to CRD", Subcommands: migrateSubCommands},
}
app.Run(os.Args)
fission/migrate.go
+213
View File
@@ -0,0 +1,213 @@
/*
Copyright 2017 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"github.com/urfave/cli"
"github.com/fission/fission"
"github.com/fission/fission/controller/client"
"github.com/fission/fission/crd"
"github.com/fission/fission/mqtrigger/messageQueue"
)
type (
TPRResource struct {
Packages []crd.Package `json:"packages"`
Functions []crd.Function `json:"functions"`
Environments []crd.Environment `json:"environments"`
HTTPTriggers []crd.HTTPTrigger `json:"httptriggers"`
Mqtriggers []crd.MessageQueueTrigger `json:"mqtriggers"`
TimeTriggers []crd.TimeTrigger `json:"timetriggers"`
Watches []crd.KubernetesWatchTrigger `json:"watches"`
}
)
func migrateDumpTPRResource(client *client.Client, filename string) {
pkgs, err := client.PackageList()
checkErr(err, "dump packages")
fns, err := client.FunctionList()
checkErr(err, "dump functions")
httpTriggers, err := client.HTTPTriggerList()
checkErr(err, "dump http triggers")
envs, err := client.EnvironmentList()
checkErr(err, "dump environments")
watches, err := client.WatchList()
checkErr(err, "dump watches")
timeTriggers, err := client.TimeTriggerList()
checkErr(err, "dump time triggers")
mqTriggers, err := client.MessageQueueTriggerList(messageQueue.NATS)
checkErr(err, "dump message queue triggers")
tprResource := TPRResource{
Packages: pkgs,
Functions: fns,
HTTPTriggers: httpTriggers,
Environments: envs,
Watches: watches,
TimeTriggers: timeTriggers,
Mqtriggers: mqTriggers,
}
// serialize tprResource
out, err := json.MarshalIndent(tprResource, "", " ")
checkErr(err, "serialize tpr state")
// dump to file fission-tpr.json
if len(filename) == 0 {
filename = "fission-tpr.json"
}
err = ioutil.WriteFile(filename, out, 0644)
checkErr(err, "write file")
fmt.Printf("Done: Saved %v packages, %v functions, %v HTTP triggers, %v watches, %v message queue triggers, %v time triggers.\n",
len(tprResource.Packages), len(tprResource.Functions), len(tprResource.HTTPTriggers), len(tprResource.Watches), len(tprResource.Mqtriggers),
len(tprResource.TimeTriggers))
}
func migrateDumpTPR(c *cli.Context) error {
filename := c.String("file")
client := getClient(c.GlobalString("server"))
migrateDumpTPRResource(client, filename)
return nil
}
func migrateDeleteTPR(c *cli.Context) error {
server := c.GlobalString("server")
relativeUrl := fmt.Sprintf("%v/%v", server, "v2/deleteTpr")
req, err := http.NewRequest("DELETE", relativeUrl, nil)
checkErr(err, "connect to fission server")
resp, err := http.DefaultClient.Do(req)
checkErr(err, "delete tpr resources")
defer resp.Body.Close()
if resp.StatusCode == 404 {
msg := fmt.Sprintf("Server %v isn't support deleteTpr method. Use --server to point at a 0.4.0+ Fission server.", server)
fatal(msg)
}
return nil
}
// checkAlreadyExistsError helps to check whether the error is AlreadyExists error or not.
func checkAlreadyExistsError(err error, msg string) {
fe, ok := err.(fission.Error)
// ignore AlreadyExists error, since a resource may exist.
if !ok || fe.Code != fission.ErrorNameExists {
checkErr(err, msg)
}
}
func migrateRestoreCRD(c *cli.Context) error {
filename := c.String("file")
if len(filename) == 0 {
filename = "fission-tpr.json"
}
contents, err := ioutil.ReadFile(filename)
checkErr(err, fmt.Sprintf("open file %v", filename))
var tprResource TPRResource
err = json.Unmarshal(contents, &tprResource)
checkErr(err, "parse dumped tpr")
client := getClient(c.GlobalString("server"))
// Though Kubernetes will migrate TPRs to CRDs automatically when TPR definition is
// deleted if the same name CRD exists. We still need to make sure that there is no
// resource gets lost during the migration. Also, since we changed the capitalization
// of some CRDs to CamelCase (e.g. Httptrigger -> HTTPTrigger), we need to recreate
// those resources by ourselves.
// create envs
for _, e := range tprResource.Environments {
e.Metadata.ResourceVersion = ""
_, err = client.EnvironmentCreate(&crd.Environment{
Metadata: e.Metadata,
Spec: e.Spec,
})
checkAlreadyExistsError(err, fmt.Sprintf("create environment %v", e.Metadata.Name))
}
// create httptriggers
for _, t := range tprResource.HTTPTriggers {
t.Metadata.ResourceVersion = ""
_, err = client.HTTPTriggerCreate(&crd.HTTPTrigger{
Metadata: t.Metadata,
Spec: t.Spec,
})
checkAlreadyExistsError(err, fmt.Sprintf("create http trigger %v", t.Metadata.Name))
}
// create mqtriggers
for _, t := range tprResource.Mqtriggers {
t.Metadata.ResourceVersion = ""
_, err = client.MessageQueueTriggerCreate(&crd.MessageQueueTrigger{
Metadata: t.Metadata,
Spec: t.Spec,
})
checkAlreadyExistsError(err, fmt.Sprintf("create http trigger %v", t.Metadata.Name))
}
// create time triggers
for _, t := range tprResource.TimeTriggers {
t.Metadata.ResourceVersion = ""
_, err = client.TimeTriggerCreate(&crd.TimeTrigger{
Metadata: t.Metadata,
Spec: t.Spec,
})
checkAlreadyExistsError(err, fmt.Sprintf("create time trigger %v", t.Metadata.Name))
}
// create watches
for _, t := range tprResource.Watches {
t.Metadata.ResourceVersion = ""
_, err = client.WatchCreate(&crd.KubernetesWatchTrigger{
Metadata: t.Metadata,
Spec: t.Spec,
})
checkAlreadyExistsError(err, fmt.Sprintf("create kubernetes watch trigger %v", t.Metadata.Name))
}
// create packages
for _, p := range tprResource.Packages {
p.Metadata.ResourceVersion = ""
_, err = client.PackageCreate(&crd.Package{
Metadata: p.Metadata,
Spec: p.Spec,
})
checkAlreadyExistsError(err, fmt.Sprintf("create function %v", p.Metadata.Name))
}
// create functions
for _, f := range tprResource.Functions {
f.Metadata.ResourceVersion = ""
_, err = client.FunctionCreate(&crd.Function{
Metadata: f.Metadata,
Spec: f.Spec,
})
checkAlreadyExistsError(err, fmt.Sprintf("create function %v", f.Metadata.Name))
}
return nil
}
fission/mqtrigger.go
+2 -2
View File
@@ -26,8 +26,8 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/crd"
"github.com/fission/fission/mqtrigger/messageQueue"
"github.com/fission/fission/tpr"
)
func mqtCreate(c *cli.Context) error {
@@ -72,7 +72,7 @@ func mqtCreate(c *cli.Context) error {
checkMQTopicAvailability(mqType, topic, respTopic)
mqt := tpr.Messagequeuetrigger{
mqt := crd.MessageQueueTrigger{
Metadata: metav1.ObjectMeta{
Name: mqtName,
Namespace: metav1.NamespaceDefault,
fission/timetrigger.go
+2 -2
View File
@@ -26,7 +26,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/tpr"
"github.com/fission/fission/crd"
)
func ttCreate(c *cli.Context) error {
@@ -45,7 +45,7 @@ func ttCreate(c *cli.Context) error {
fatal("Need a cron spec like '0 30 * * *', '@every 1h30m', or '@hourly'; use --cron")
}
tt := &tpr.Timetrigger{
tt := &crd.TimeTrigger{
Metadata: metav1.ObjectMeta{
Name: name,
Namespace: metav1.NamespaceDefault,
fission/upgrade.go
+24 -24
View File
@@ -15,7 +15,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/tpr"
"github.com/fission/fission/crd"
"github.com/fission/fission/v1"
)
@@ -23,9 +23,9 @@ type (
V1FissionState struct {
Functions []v1.Function `json:"functions"`
Environments []v1.Environment `json:"environments"`
Httptriggers []v1.HTTPTrigger `json:"httptriggers"`
HTTPTriggers []v1.HTTPTrigger `json:"httptriggers"`
Mqtriggers []v1.MessageQueueTrigger `json:"mqtriggers"`
Timetriggers []v1.TimeTrigger `json:"timetriggers"`
TimeTriggers []v1.TimeTrigger `json:"timetriggers"`
Watches []v1.Watch `json:"watches"`
NameChanges map[string]string `json:"namechanges"`
}
@@ -133,7 +133,7 @@ func upgradeDumpV1State(v1url string, filename string) {
fmt.Println("Getting routes")
resp = get(v1url + "/triggers/http")
err = json.Unmarshal(resp, &v1state.Httptriggers)
err = json.Unmarshal(resp, &v1state.HTTPTriggers)
checkErr(err, "parse server response")
fmt.Println("Getting message queue triggers")
@@ -143,7 +143,7 @@ func upgradeDumpV1State(v1url string, filename string) {
fmt.Println("Getting time triggers")
resp = get(v1url + "/triggers/time")
err = json.Unmarshal(resp, &v1state.Timetriggers)
err = json.Unmarshal(resp, &v1state.TimeTriggers)
checkErr(err, "parse server response")
fmt.Println("Getting function list")
@@ -163,7 +163,7 @@ func upgradeDumpV1State(v1url string, filename string) {
funcMetaSet[f.Metadata] = true
nr.trackName(f.Metadata.Name)
}
for _, t := range v1state.Httptriggers {
for _, t := range v1state.HTTPTriggers {
funcMetaSet[t.Function] = true
nr.trackName(t.Metadata.Name)
}
@@ -175,7 +175,7 @@ func upgradeDumpV1State(v1url string, filename string) {
funcMetaSet[t.Function] = true
nr.trackName(t.Metadata.Name)
}
for _, t := range v1state.Timetriggers {
for _, t := range v1state.TimeTriggers {
funcMetaSet[t.Function] = true
nr.trackName(t.Metadata.Name)
}
@@ -225,7 +225,7 @@ func upgradeDumpV1State(v1url string, filename string) {
checkErr(err, "write file")
fmt.Printf("Done: Saved %v functions, %v HTTP triggers, %v watches, %v message queue triggers, %v time triggers.\n",
len(v1state.Functions), len(v1state.Httptriggers), len(v1state.Watches), len(v1state.Mqtriggers), len(v1state.Timetriggers))
len(v1state.Functions), len(v1state.HTTPTriggers), len(v1state.Watches), len(v1state.Mqtriggers), len(v1state.TimeTriggers))
}
func functionRefFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *fission.FunctionReference {
@@ -235,7 +235,7 @@ func functionRefFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *fis
}
}
func tprMetadataFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *metav1.ObjectMeta {
func crdMetadataFromV1Metadata(m *v1.Metadata, nameRemap map[string]string) *metav1.ObjectMeta {
return &metav1.ObjectMeta{
Name: nameRemap[m.Name],
Namespace: metav1.NamespaceDefault,
@@ -302,7 +302,7 @@ func upgradeRestoreState(c *cli.Context) error {
},
Deployment: *archive,
}
pkg, err := client.PackageCreate(&tpr.Package{
pkg, err := client.PackageCreate(&crd.Package{
Metadata: metav1.ObjectMeta{
Name: pkgName,
Namespace: metav1.NamespaceDefault,
@@ -310,8 +310,8 @@ func upgradeRestoreState(c *cli.Context) error {
Spec: pkgSpec,
})
checkErr(err, fmt.Sprintf("create package %v", pkgName))
_, err = client.FunctionCreate(&tpr.Function{
Metadata: *tprMetadataFromV1Metadata(&f.Metadata, v1state.NameChanges),
_, err = client.FunctionCreate(&crd.Function{
Metadata: *crdMetadataFromV1Metadata(&f.Metadata, v1state.NameChanges),
Spec: fission.FunctionSpec{
Environment: pkgSpec.Environment,
Package: fission.FunctionPackageRef{
@@ -329,8 +329,8 @@ func upgradeRestoreState(c *cli.Context) error {
// create envs
for _, e := range v1state.Environments {
_, err = client.EnvironmentCreate(&tpr.Environment{
Metadata: *tprMetadataFromV1Metadata(&e.Metadata, v1state.NameChanges),
_, err = client.EnvironmentCreate(&crd.Environment{
Metadata: *crdMetadataFromV1Metadata(&e.Metadata, v1state.NameChanges),
Spec: fission.EnvironmentSpec{
Version: 1,
Runtime: fission.Runtime{
@@ -342,9 +342,9 @@ func upgradeRestoreState(c *cli.Context) error {
}
// create httptriggers
for _, t := range v1state.Httptriggers {
_, err = client.HTTPTriggerCreate(&tpr.Httptrigger{
Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
for _, t := range v1state.HTTPTriggers {
_, err = client.HTTPTriggerCreate(&crd.HTTPTrigger{
Metadata: *crdMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
Spec: fission.HTTPTriggerSpec{
RelativeURL: t.UrlPattern,
Method: t.Method,
@@ -356,8 +356,8 @@ func upgradeRestoreState(c *cli.Context) error {
// create mqtriggers
for _, t := range v1state.Mqtriggers {
_, err = client.MessageQueueTriggerCreate(&tpr.Messagequeuetrigger{
Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
_, err = client.MessageQueueTriggerCreate(&crd.MessageQueueTrigger{
Metadata: *crdMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
Spec: fission.MessageQueueTriggerSpec{
FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges),
MessageQueueType: t.MessageQueueType,
@@ -369,9 +369,9 @@ func upgradeRestoreState(c *cli.Context) error {
}
// create time triggers
for _, t := range v1state.Timetriggers {
_, err = client.TimeTriggerCreate(&tpr.Timetrigger{
Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
for _, t := range v1state.TimeTriggers {
_, err = client.TimeTriggerCreate(&crd.TimeTrigger{
Metadata: *crdMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
Spec: fission.TimeTriggerSpec{
FunctionReference: *functionRefFromV1Metadata(&t.Function, v1state.NameChanges),
Cron: t.Cron,
@@ -382,8 +382,8 @@ func upgradeRestoreState(c *cli.Context) error {
// create watches
for _, t := range v1state.Watches {
_, err = client.WatchCreate(&tpr.Kuberneteswatchtrigger{
Metadata: *tprMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
_, err = client.WatchCreate(&crd.KubernetesWatchTrigger{
Metadata: *crdMetadataFromV1Metadata(&t.Metadata, v1state.NameChanges),
Spec: fission.KubernetesWatchTriggerSpec{
Namespace: t.Namespace,
Type: t.ObjType,
fission/watch.go
+2 -2
View File
@@ -26,7 +26,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/tpr"
"github.com/fission/fission/crd"
)
func wCreate(c *cli.Context) error {
@@ -61,7 +61,7 @@ func wCreate(c *cli.Context) error {
// automatically name watches
watchName := uuid.NewV4().String()
w := &tpr.Kuberneteswatchtrigger{
w := &crd.KubernetesWatchTrigger{
Metadata: metav1.ObjectMeta{
Name: watchName,
Namespace: metav1.NamespaceDefault,