Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move builds to package level (#297)

Browse Source 
Introduce _Archives_ as a type to reference arbitrary blobs.

_Packages_ are a pair of Source and Deployment archives. Packages have an environment reference.

Functions reference a package. Multiple functions can reference the same package.
This commit is contained in:
Soam Vasani
2017-09-08 14:19:39 -07:00
committed by GitHub
parent 9d7bd49338
commit f2091d0d80
7 changed files with 142 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
controller/api_test.go
+5 -3
View File
@@ -81,8 +81,10 @@ func TestFunctionApi(t *testing.T) {
Namespace: api.NamespaceDefault,
},
Spec: fission.FunctionSpec{
EnvironmentName: "nodejs",
Deployment: fission.FunctionPackageRef{
Environment: fission.EnvironmentReference{
Name: "nodejs",
},
Package: fission.FunctionPackageRef{
FunctionName: "xxx",
},
},
@@ -99,7 +101,7 @@ func TestFunctionApi(t *testing.T) {
_, err = g.client.FunctionCreate(testFunc)
assertNameReuseFailure(err, "function")
testFunc.Spec.Deployment.FunctionName = "yyy"
testFunc.Spec.Package.FunctionName = "yyy"
_, err = g.client.FunctionUpdate(testFunc)
panicIf(err)
controller/packageApi.go
+7 -2
View File
@@ -65,7 +65,12 @@ func (a *API) PackageApiCreate(w http.ResponseWriter, r *http.Request) {
}
// Ensure size limits
if len(f.Spec.Literal) > 256*1024 {
if len(f.Spec.Source.Literal) > 256*1024 {
err := fission.MakeError(fission.ErrorInvalidArgument, "Package literal larger than 256K")
a.respondWithError(w, err)
return
}
if len(f.Spec.Deployment.Literal) > 256*1024 {
err := fission.MakeError(fission.ErrorInvalidArgument, "Package literal larger than 256K")
a.respondWithError(w, err)
return
@@ -104,7 +109,7 @@ func (a *API) PackageApiGet(w http.ResponseWriter, r *http.Request) {
var resp []byte
if raw != "" {
resp = []byte(f.Spec.Literal)
resp = []byte(f.Spec.Deployment.Literal)
} else {
resp, err = json.Marshal(f)
if err != nil {
environments/fetcher/fetcher.go
+13 -9
View File
@@ -155,11 +155,8 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
// get pkg
var pkg *tpr.Package
if req.FetchType == FETCH_SOURCE {
pkg, err = fetcher.fissionClient.Packages(fn.Spec.Source.PackageRef.Namespace).Get(fn.Spec.Source.PackageRef.Name)
} else if req.FetchType == FETCH_DEPLOYMENT {
pkg, err = fetcher.fissionClient.Packages(fn.Spec.Deployment.PackageRef.Namespace).Get(fn.Spec.Deployment.PackageRef.Name)
}
pkg, err = fetcher.fissionClient.
Packages(fn.Spec.Package.PackageRef.Namespace).Get(fn.Spec.Package.PackageRef.Name)
if err != nil {
e := fmt.Sprintf("Failed to get package: %v", err)
log.Printf(e)
@@ -167,10 +164,17 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
return
}
var archive *fission.Archive
if req.FetchType == FETCH_SOURCE {
archive = &pkg.Spec.Source
} else if req.FetchType == FETCH_DEPLOYMENT {
archive = &pkg.Spec.Deployment
}
// get package data as literal or by url
if len(pkg.Spec.Literal) > 0 {
if len(archive.Literal) > 0 {
// write pkg.Literal into tmpPath
err = ioutil.WriteFile(tmpPath, pkg.Spec.Literal, 0600)
err = ioutil.WriteFile(tmpPath, archive.Literal, 0600)
if err != nil {
e := fmt.Sprintf("Failed to write file %v: %v", tmpPath, err)
log.Printf(e)
@@ -180,7 +184,7 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
} else {
// download and verify
err = downloadUrl(pkg.Spec.URL, tmpPath)
err = downloadUrl(archive.URL, tmpPath)
if err != nil {
e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err)
log.Printf(e)
@@ -188,7 +192,7 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) {
return
}
err = verifyChecksum(tmpPath, &pkg.Spec.Checksum)
err = verifyChecksum(tmpPath, &archive.Checksum)
if err != nil {
e := fmt.Sprintf("Failed to verify checksum: %v", err)
log.Printf(e)
fission/function.go
+60 -83
View File
@@ -18,7 +18,6 @@ package main
import (
"context"
"errors"
"fmt"
"io/ioutil"
"os"
@@ -44,14 +43,13 @@ func fileSize(filePath string) int64 {
return info.Size()
}
// updatePackageSpecWithFile uploads or serializes the file into the
// provided package object
func updatePackageSpecWithFile(client *client.Client, pkgSpec *fission.PackageSpec, fileName string) {
if fileSize(fileName) < fission.PackageLiteralSizeLimit {
pkgContents := getPackageContents(fileName)
pkgSpec.Type = fission.PackageTypeLiteral
pkgSpec.Literal = pkgContents
// upload a file and return a fission.Archive
func createArchive(client *client.Client, fileName string) *fission.Archive {
var archive fission.Archive
if fileSize(fileName) < fission.ArchiveLiteralSizeLimit {
contents := getContents(fileName)
archive.Type = fission.ArchiveTypeLiteral
archive.Literal = contents
} else {
u := strings.TrimSuffix(client.Url, "/") + "/proxy/storage"
ssClient := storageSvcClient.MakeClient(u)
@@ -62,54 +60,13 @@ func updatePackageSpecWithFile(client *client.Client, pkgSpec *fission.PackageSp
archiveUrl := ssClient.GetUrl(id)
pkgSpec.Type = fission.PackageTypeUrl
pkgSpec.URL = archiveUrl
archive.Type = fission.ArchiveTypeUrl
archive.URL = archiveUrl
}
return &archive
}
// createPackageFromFile is a function that helps to upload the content
// of given file to controller to create a TPR package resource, and then
// return a function package reference for further usage.
func createPackageFromFile(client *client.Client, fnName string, fileName string) fission.FunctionPackageRef {
pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6)))
pkg := &tpr.Package{
Metadata: api.ObjectMeta{
Name: pkgName,
Namespace: api.NamespaceDefault,
},
}
updatePackageSpecWithFile(client, &pkg.Spec, fileName)
_, err := client.PackageCreate(pkg)
checkErr(err, "upload package")
return fission.FunctionPackageRef{
PackageRef: fission.PackageRef{
Name: pkgName,
Namespace: pkg.Metadata.Namespace,
},
}
}
// updatePackageContents is a function that reads content from given file
// and updates the package content of TPR package resource.
func updatePackageContents(client *client.Client, pkgName string, fileName string) error {
pkg, err := client.PackageGet(&api.ObjectMeta{
Name: pkgName,
Namespace: api.NamespaceDefault,
})
if err != nil {
return errors.New(fmt.Sprintf("read package '%v'", pkgName))
}
updatePackageSpecWithFile(client, &pkg.Spec, fileName)
_, err = client.PackageUpdate(pkg)
return err
}
func getPackageContents(filePath string) []byte {
func getContents(filePath string) []byte {
var code []byte
var err error
@@ -148,18 +105,39 @@ func fnCreate(c *cli.Context) error {
Namespace: api.NamespaceDefault,
},
Spec: fission.FunctionSpec{
EnvironmentName: envName,
Environment: fission.EnvironmentReference{
Name: envName,
Namespace: api.NamespaceDefault,
},
},
}
var pkgSpec fission.PackageSpec
if len(srcPkgName) > 0 {
function.Spec.Source = createPackageFromFile(client, fnName, srcPkgName)
pkgSpec.Source = *createArchive(client, srcPkgName)
}
if len(deployPkgName) > 0 {
function.Spec.Deployment = createPackageFromFile(client, fnName, deployPkgName)
pkgSpec.Deployment = *createArchive(client, deployPkgName)
}
pkgName := fmt.Sprintf("%v-%v", fnName, strings.ToLower(uniuri.NewLen(6)))
pkg := &tpr.Package{
Metadata: api.ObjectMeta{
Name: pkgName,
Namespace: api.NamespaceDefault,
},
Spec: pkgSpec,
}
newpkg, err := client.PackageCreate(pkg)
checkErr(err, "create package")
_, err := client.FunctionCreate(function)
function.Spec.Package = fission.FunctionPackageRef{
PackageRef: fission.PackageRef{
Name: newpkg.Name,
Namespace: newpkg.Namespace,
ResourceVersion: newpkg.ResourceVersion,
},
}
_, err = client.FunctionCreate(function)
checkErr(err, "create function")
fmt.Printf("function '%v' created\n", fnName)
@@ -210,12 +188,12 @@ func fnGet(c *cli.Context) error {
checkErr(err, "get function")
pkg, err := client.PackageGet(&api.ObjectMeta{
Name: fn.Spec.Deployment.PackageRef.Name,
Namespace: fn.Spec.Deployment.PackageRef.Namespace,
Name: fn.Spec.Package.PackageRef.Name,
Namespace: fn.Spec.Package.PackageRef.Namespace,
})
checkErr(err, "get package")
os.Stdout.Write(pkg.Spec.Literal)
os.Stdout.Write(pkg.Spec.Deployment.Literal)
return err
}
@@ -238,7 +216,7 @@ func fnGetMeta(c *cli.Context) error {
w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "ENV")
fmt.Fprintf(w, "%v\t%v\t%v\n",
f.Metadata.Name, f.Metadata.UID, f.Spec.EnvironmentName)
f.Metadata.Name, f.Metadata.UID, f.Spec.Environment.Name)
w.Flush()
return err
}
@@ -268,31 +246,30 @@ func fnUpdate(c *cli.Context) error {
fatal("Need --env or --code or --package or --srcpkg argument.")
}
// Now builder manager only starts a build if a function has a source package
// but no deployment package. This behavior will be changed after we move builds
// to package level (https://github.com/fission/fission/pull/297).
if len(srcPkgName) > 0 {
// Check the existence of the package, create it if not exist.
if len(function.Spec.Source.PackageRef.Name) > 0 {
err := updatePackageContents(client, function.Spec.Source.PackageRef.Name, srcPkgName)
checkErr(err, "update source package")
} else {
function.Spec.Source = createPackageFromFile(client, fnName, srcPkgName)
if len(deployPkgName) != 0 || len(srcPkgName) != 0 {
// get existing package
pkg, err := client.PackageGet(&api.ObjectMeta{
Name: function.Spec.Package.PackageRef.Name,
Namespace: function.Spec.Package.PackageRef.Namespace,
})
// update package spec
if len(srcPkgName) > 0 {
archive := createArchive(client, srcPkgName)
pkg.Spec.Source = *archive
}
}
if len(deployPkgName) > 0 {
if len(function.Spec.Deployment.PackageRef.Name) > 0 {
err := updatePackageContents(client, function.Spec.Deployment.PackageRef.Name, deployPkgName)
checkErr(err, "update source package")
} else {
function.Spec.Deployment = createPackageFromFile(client, fnName, deployPkgName)
if len(deployPkgName) > 0 {
archive := createArchive(client, deployPkgName)
pkg.Spec.Deployment = *archive
}
// updage package object
newpkg, err := client.PackageUpdate(pkg)
checkErr(err, "update package")
// update function spec with resource version
function.Spec.Package.PackageRef.ResourceVersion = newpkg.ResourceVersion
}
if len(envName) > 0 {
function.Spec.EnvironmentName = envName
function.Spec.Environment.Name = envName
}
_, err = client.FunctionUpdate(function)
@@ -333,7 +310,7 @@ func fnList(c *cli.Context) error {
fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "ENV")
for _, f := range fns {
fmt.Fprintf(w, "%v\t%v\t%v\n",
f.Metadata.Name, f.Metadata.UID, f.Spec.EnvironmentName)
f.Metadata.Name, f.Metadata.UID, f.Spec.Environment.Name)
}
w.Flush()
poolmgr/api.go
+1 -1
View File
@@ -167,7 +167,7 @@ func (poolMgr *Poolmgr) getFunctionEnv(m *api.ObjectMeta) (*tpr.Environment, err
// Get env from metadata
log.Printf("[%v] getting env", m)
env, err = poolMgr.fissionClient.Environments(f.Metadata.Namespace).Get(f.Spec.EnvironmentName)
env, err = poolMgr.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
if err != nil {
return nil, err
}
tpr/tpr_test.go
+8 -7
View File
@@ -46,15 +46,16 @@ func functionTests(tprClient *rest.RESTClient) {
Name: "hello",
},
Spec: fission.FunctionSpec{
Source: fission.FunctionPackageRef{},
Deployment: fission.FunctionPackageRef{
Package: fission.FunctionPackageRef{
PackageRef: fission.PackageRef{
Name: "foo",
Namespace: "bar",
},
FunctionName: "hello",
},
EnvironmentName: "xxx",
Environment: fission.EnvironmentReference{
Name: "xxx",
},
},
}
@@ -74,14 +75,14 @@ func functionTests(tprClient *rest.RESTClient) {
// read
f, err = fi.Get(function.Metadata.Name)
panicIf(err)
if f.Spec.Deployment.FunctionName != function.Spec.Deployment.FunctionName {
if f.Spec.Environment.Name != function.Spec.Environment.Name {
log.Panicf("Bad result from Get: %v", f)
}
log.Printf("f.Metadata = %#v", f.Metadata)
// update
function.Spec.EnvironmentName = "yyy"
function.Spec.Environment.Name = "yyy"
f, err = fi.Update(function)
panicIf(err)
@@ -93,7 +94,7 @@ func functionTests(tprClient *rest.RESTClient) {
if len(fl.Items) != 1 {
log.Panicf("wrong count from list: %v", fl)
}
if fl.Items[0].Spec.EnvironmentName != function.Spec.EnvironmentName {
if fl.Items[0].Spec.Environment.Name != function.Spec.Environment.Name {
log.Panicf("bad object from list: %v", fl.Items[0])
}
@@ -122,7 +123,7 @@ func functionTests(tprClient *rest.RESTClient) {
if !ok {
log.Panicf("Can't cast to Function")
}
if wf.Spec.EnvironmentName != function.Spec.EnvironmentName {
if wf.Spec.Environment.Name != function.Spec.Environment.Name {
log.Panicf("Bad object from watch: %#v", wf)
}
log.Printf("watch event took %v", time.Now().Sub(start))
types.go
+48 -24
View File
@@ -34,16 +34,16 @@ type (
Sum string `json:"sum"`
}
// PackageType is either literal or URL, indicating whether
// the package is specified in the Package struct or
// ArchiveType is either literal or URL, indicating whether
// the package is specified in the Archive struct or
// externally.
PackageType string
ArchiveType string
// Package contains or references a collection of source or
// binary files.
PackageSpec struct {
Archive struct {
// Type defines how the package is specified: literal or URL.
Type PackageType `json:"type"`
Type ArchiveType `json:"type"`
// Literal contents of the package. Can be used for
// encoding packages below TODO (256KB?) size.
@@ -57,12 +57,34 @@ type (
Checksum Checksum `json:"checksum"`
}
EnvironmentReference struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
}
BuildStatus string
PackageSpec struct {
Environment EnvironmentReference `json:"environment"`
Source Archive `json:"source"`
Deployment Archive `json:"deployment"`
// In the future, we can have a debug build here too
}
PackageStatus struct {
BuildStatus BuildStatus `json:"buildstatus"`
BuildLog string `json:"buildlog"` // output of the build (errors etc)
}
PackageRef struct {
Name string
Namespace string
Namespace string `json:"namespace"`
Name string `json:"name"`
// Including resource version in the reference forces the function to be updated on
// package update, making it possible to cache the function based on its metadata.
ResourceVersion string `json:"resourceversion"`
}
FunctionPackageRef struct {
PackageRef PackageRef
PackageRef PackageRef `json:"packageref"`
// FunctionName specifies a specific function within the package. This allows
// functions to share packages, by having different functions within the same
@@ -77,18 +99,13 @@ type (
// FunctionSpec describes the contents of the function.
FunctionSpec struct {
// EnvironmentName is the name of the environment that this function is associated
// with. An Environment with this name should exist, otherwise the function cannot
// be invoked.
EnvironmentName string `json:"environmentName"`
// Environment is the build and runtime environment that this function is
// associated with. An Environment with this name should exist, otherwise the
// function cannot be invoked.
Environment EnvironmentReference `json:"environment"`
// Source is an source package for this function; it's used for the build step if
// the environment defines a build container.
Source FunctionPackageRef `json:"source"`
// Deployment is a deployable package for this function. This is the package that's
// loaded into the environment's runtime container.
Deployment FunctionPackageRef `json:"deployment"`
// Reference to a package containing deployment and optionally the source
Package FunctionPackageRef `json:"package"`
}
FunctionReferenceType string
@@ -220,12 +237,19 @@ const (
)
const (
// PackageTypeLiteral means the package contents are specified in the Literal field of
// ArchiveTypeLiteral means the package contents are specified in the Literal field of
// resource itself.
PackageTypeLiteral PackageType = "literal"
ArchiveTypeLiteral ArchiveType = "literal"
// PackageTypeUrl means the package contents are at the specified URL.
PackageTypeUrl PackageType = "url"
// ArchiveTypeUrl means the package contents are at the specified URL.
ArchiveTypeUrl ArchiveType = "url"
)
const (
BuildStatusPending = "pending"
BuildStatusRunning = "running"
BuildStatusSucceeded = "succeeded"
BuildStatusFailed = "failed"
)
const (
@@ -266,5 +290,5 @@ var errorDescriptions = []string{
}
const (
PackageLiteralSizeLimit int64 = 256 * 1024
ArchiveLiteralSizeLimit int64 = 256 * 1024
)