diff --git a/pkg/fetcher/fetcher.go b/pkg/fetcher/fetcher.go index 4bb5b0be..efca4562 100644 --- a/pkg/fetcher/fetcher.go +++ b/pkg/fetcher/fetcher.go @@ -21,7 +21,6 @@ import ( "context" "encoding/json" "fmt" - "io" "io/ioutil" "net/http" "os" @@ -33,7 +32,6 @@ import ( uuid "github.com/satori/go.uuid" "go.opencensus.io/plugin/ochttp" "go.uber.org/zap" - "golang.org/x/net/context/ctxhttp" k8serr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -96,38 +94,6 @@ func MakeFetcher(logger *zap.Logger, sharedVolumePath string, sharedSecretPath s }, nil } -func downloadUrl(ctx context.Context, httpClient *http.Client, url string, localPath string) error { - resp, err := ctxhttp.Get(ctx, httpClient, url) - if err != nil { - return err - } - defer resp.Body.Close() - - w, err := os.Create(localPath) - if err != nil { - return err - } - defer w.Close() - - _, err = io.Copy(w, resp.Body) - if err != nil { - return err - } - - // flushing write buffer to file - err = w.Sync() - if err != nil { - return err - } - - err = os.Chmod(localPath, 0600) - if err != nil { - return err - } - - return nil -} - func verifyChecksum(fileChecksum, checksum *fv1.Checksum) error { if checksum.Type != fv1.ChecksumTypeSHA256 { return ferror.MakeError(ferror.ErrorInvalidArgument, "Unsupported checksum type") @@ -263,7 +229,7 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, pkg *fv1.Package, req types.F if req.FetchType == types.FETCH_URL { // fetch the file and save it to the tmp path - err := downloadUrl(ctx, fetcher.httpClient, req.Url, tmpPath) + err := utils.DownloadUrl(ctx, fetcher.httpClient, req.Url, tmpPath) if err != nil { e := "failed to download url" fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url)) @@ -302,7 +268,7 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, pkg *fv1.Package, req types.F } } else { // download and verify - err := downloadUrl(ctx, fetcher.httpClient, archive.URL, tmpPath) + err := utils.DownloadUrl(ctx, fetcher.httpClient, archive.URL, tmpPath) if err != nil { e := "failed to download url" fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url)) diff --git a/pkg/fission-cli/cmd/function/command.go b/pkg/fission-cli/cmd/function/command.go index 8a4502e3..59dcc615 100644 --- a/pkg/fission-cli/cmd/function/command.go +++ b/pkg/fission-cli/cmd/function/command.go @@ -36,13 +36,16 @@ func Commands() *cobra.Command { flag.FnExecutorType, flag.FnCfgMap, flag.FnSecret, flag.FnSpecializationTimeout, flag.FnExecutionTimeout, - // TODO retired pkg related flag from function cmd + // TODO retired pkg & trigger related flags from function cmd flag.PkgCode, flag.PkgSrcArchive, flag.PkgDeployArchive, + flag.PkgSrcChecksum, flag.PkgDeployChecksum, flag.PkgInsecure, + flag.FnBuildCmd, + flag.HtUrl, flag.HtMethod, // flag for newdeploy to use. flag.RunTimeMinCPU, flag.RunTimeMaxCPU, flag.RunTimeMinMemory, - flag.RunTimeMaxMemory, flag.FnBuildCmd, flag.ReplicasMin, + flag.RunTimeMaxMemory, flag.ReplicasMin, flag.ReplicasMax, flag.RunTimeTargetCPU, flag.NamespaceFunction, flag.NamespaceEnvironment, flag.SpecSave}, @@ -84,6 +87,7 @@ func Commands() *cobra.Command { flag.FnSpecializationTimeout, flag.FnExecutionTimeout, flag.PkgCode, flag.PkgSrcArchive, flag.PkgDeployArchive, + flag.PkgSrcChecksum, flag.PkgDeployChecksum, flag.PkgInsecure, flag.FnBuildCmd, flag.PkgForce, flag.RunTimeMinCPU, flag.RunTimeMaxCPU, flag.RunTimeMinMemory, diff --git a/pkg/fission-cli/cmd/function/update.go b/pkg/fission-cli/cmd/function/update.go index 8265a466..1c541601 100644 --- a/pkg/fission-cli/cmd/function/update.go +++ b/pkg/fission-cli/cmd/function/update.go @@ -82,31 +82,14 @@ func (opts *UpdateSubCommand) complete(input cli.Input) error { envNamespace = "" } - var deployArchiveFiles []string - codeFlag := false - code := input.String(flagkey.PkgCode) - if len(code) == 0 { - deployArchiveFiles = input.StringSlice(flagkey.PkgDeployArchive) - } else { - deployArchiveFiles = append(deployArchiveFiles, input.String(flagkey.PkgCode)) - codeFlag = true - } - - srcArchiveFiles := input.StringSlice(flagkey.PkgSrcArchive) pkgName := input.String(flagkey.FnPackageName) entrypoint := input.String(flagkey.FnEntrypoint) - buildcmd := input.String(flagkey.PkgBuildCmd) - force := input.Bool(flagkey.PkgForce) secretNames := input.StringSlice(flagkey.FnSecret) cfgMapNames := input.StringSlice(flagkey.FnCfgMap) specializationTimeout := input.Int(flagkey.FnSpecializationTimeout) - if len(srcArchiveFiles) > 0 && len(deployArchiveFiles) > 0 { - return errors.Errorf("need either of --%v or --%v and not both arguments", flagkey.PkgSrcArchive, flagkey.PkgDeployArchive) - } - var secrets []fv1.SecretReference var configMaps []fv1.ConfigMapReference @@ -214,36 +197,38 @@ func (opts *UpdateSubCommand) complete(input cli.Input) error { return errors.Wrap(err, fmt.Sprintf("read package '%v.%v'. Pkg should be present in the same ns as the function", pkgName, fnNamespace)) } - pkgMetadata := &pkg.Metadata + forceUpdate := input.Bool(flagkey.PkgForce) - if len(deployArchiveFiles) != 0 || len(srcArchiveFiles) != 0 || len(buildcmd) != 0 || len(envName) != 0 || len(envNamespace) != 0 { - fnList, err := _package.GetFunctionsByPackage(opts.client, pkg.Metadata.Name, pkg.Metadata.Namespace) - if err != nil { - return errors.Wrap(err, "error getting function list") - } + fnList, err := _package.GetFunctionsByPackage(opts.client, pkg.Metadata.Name, pkg.Metadata.Namespace) + if err != nil { + return errors.Wrap(err, "error getting function list") + } - if !force && len(fnList) > 1 { - return errors.New("package is used by multiple functions, use --force to force update") - } + if !forceUpdate && len(fnList) > 1 { + return errors.Errorf("Package is used by multiple functions, use --%v to force update", flagkey.PkgForce) + } - pkgMetadata, err = _package.UpdatePackage(opts.client, pkg, envName, envNamespace, srcArchiveFiles, deployArchiveFiles, buildcmd, false, codeFlag) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("error updating package '%v'", pkgName)) - } + newPkgMeta, err := _package.UpdatePackage(input, opts.client, pkg) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("error updating package '%v'", pkgName)) + } - fmt.Printf("package '%v' updated\n", pkgMetadata.GetName()) - - // update resource version of package reference of functions that shared the same package + // the package resource version of function has been changed, + // we need to update function resource version to prevent conflict. + // TODO: remove this block when deprecating pkg flags of function command. + if pkg.Metadata.ResourceVersion != newPkgMeta.ResourceVersion { + var fns []fv1.Function + // don't update the package resource version of the function we are currently + // updating to prevent update conflict. for _, fn := range fnList { - // ignore the update for current function here, it will be updated later. - if fn.Metadata.Name != fnName { - fn.Spec.Package.PackageRef.ResourceVersion = pkgMetadata.ResourceVersion - _, err := opts.client.FunctionUpdate(&fn) - if err != nil { - return errors.Wrap(err, "error updating function") - } + if fn.Metadata.UID != function.Metadata.UID { + fns = append(fns, fn) } } + err = _package.UpdateFunctionPackageResourceVersion(opts.client, newPkgMeta, fns...) + if err != nil { + return errors.Wrap(err, "error updating function package reference resource version") + } } // TODO : One corner case where user just updates the pkg reference with fnUpdate, but internally this new pkg reference @@ -251,9 +236,9 @@ func (opts *UpdateSubCommand) complete(input cli.Input) error { // update function spec with new package metadata function.Spec.Package.PackageRef = fv1.PackageRef{ - Namespace: pkgMetadata.Namespace, - Name: pkgMetadata.Name, - ResourceVersion: pkgMetadata.ResourceVersion, + Namespace: newPkgMeta.Namespace, + Name: newPkgMeta.Name, + ResourceVersion: newPkgMeta.ResourceVersion, } if function.Spec.Environment.Name != pkg.Spec.Environment.Name { @@ -273,6 +258,6 @@ func (opts *UpdateSubCommand) run(input cli.Input) error { return errors.Wrap(err, "error updating function") } - fmt.Printf("function '%v' updated\n", opts.function.Metadata.Name) + fmt.Printf("Function '%v' updated\n", opts.function.Metadata.Name) return nil } diff --git a/pkg/fission-cli/cmd/package/command.go b/pkg/fission-cli/cmd/package/command.go index f414d028..330df60c 100644 --- a/pkg/fission-cli/cmd/package/command.go +++ b/pkg/fission-cli/cmd/package/command.go @@ -32,7 +32,8 @@ func Commands() *cobra.Command { wrapper.SetFlags(createCmd, flag.FlagSet{ Required: []flag.Flag{flag.PkgEnvironment}, Optional: []flag.Flag{flag.PkgName, flag.PkgCode, flag.PkgSrcArchive, flag.PkgDeployArchive, - flag.PkgBuildCmd, flag.NamespacePackage, flag.NamespaceEnvironment, flag.SpecSave}, + flag.PkgSrcChecksum, flag.PkgDeployChecksum, flag.PkgInsecure, flag.PkgBuildCmd, + flag.NamespacePackage, flag.NamespaceEnvironment, flag.SpecSave}, }) getSrcCmd := &cobra.Command{ @@ -62,8 +63,9 @@ func Commands() *cobra.Command { } wrapper.SetFlags(updateCmd, flag.FlagSet{ Required: []flag.Flag{flag.PkgName}, - Optional: []flag.Flag{flag.PkgEnvironment, flag.PkgSrcArchive, flag.PkgDeployArchive, - flag.PkgBuildCmd, flag.PkgForce, flag.NamespacePackage, flag.NamespaceEnvironment}, + Optional: []flag.Flag{flag.PkgEnvironment, flag.PkgCode, flag.PkgSrcArchive, flag.PkgDeployArchive, + flag.PkgSrcChecksum, flag.PkgDeployChecksum, flag.PkgInsecure, flag.PkgBuildCmd, flag.PkgForce, + flag.NamespacePackage, flag.NamespaceEnvironment}, }) deleteCmd := &cobra.Command{ diff --git a/pkg/fission-cli/cmd/package/create.go b/pkg/fission-cli/cmd/package/create.go index dd137ef9..8034e6c1 100644 --- a/pkg/fission-cli/cmd/package/create.go +++ b/pkg/fission-cli/cmd/package/create.go @@ -124,6 +124,10 @@ func (opts *CreateSubCommand) run(input cli.Input) error { func CreatePackage(input cli.Input, client *client.Client, pkgName string, pkgNamespace string, envName string, envNamespace string, srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, specDir string, specFile string, noZip bool) (*metav1.ObjectMeta, error) { + insecure := input.Bool(flagkey.PkgInsecure) + deployChecksum := input.String(flagkey.PkgDeployChecksum) + srcChecksum := input.String(flagkey.PkgSrcChecksum) + pkgSpec := fv1.PackageSpec{ Environment: fv1.EnvironmentReference{ Namespace: envNamespace, @@ -136,9 +140,9 @@ func CreatePackage(input cli.Input, client *client.Client, pkgName string, pkgNa if len(specFile) > 0 { // we should do this in all cases, i think pkgStatus = fv1.BuildStatusNone } - deployment, err := CreateArchive(client, deployArchiveFiles, noZip, specDir, specFile) + deployment, err := CreateArchive(client, deployArchiveFiles, noZip, insecure, deployChecksum, specDir, specFile) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error creating source archive") } pkgSpec.Deployment = *deployment if len(pkgName) == 0 { @@ -146,9 +150,9 @@ func CreatePackage(input cli.Input, client *client.Client, pkgName string, pkgNa } } if len(srcArchiveFiles) > 0 { - source, err := CreateArchive(client, srcArchiveFiles, false, specDir, specFile) + source, err := CreateArchive(client, srcArchiveFiles, false, insecure, srcChecksum, specDir, specFile) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error creating deploy archive") } pkgSpec.Source = *source pkgStatus = fv1.BuildStatusPending // set package build status to pending @@ -164,6 +168,7 @@ func CreatePackage(input cli.Input, client *client.Client, pkgName string, pkgNa if len(pkgName) == 0 { pkgName = strings.ToLower(uuid.NewV4().String()) } + pkg := &fv1.Package{ Metadata: metav1.ObjectMeta{ Name: pkgName, diff --git a/pkg/fission-cli/cmd/package/package.go b/pkg/fission-cli/cmd/package/package.go index 7bbda62a..afdb15e7 100644 --- a/pkg/fission-cli/cmd/package/package.go +++ b/pkg/fission-cli/cmd/package/package.go @@ -19,6 +19,7 @@ package _package import ( "context" "fmt" + "net/http" "os" "path/filepath" "strings" @@ -33,15 +34,18 @@ import ( pkgutil "github.com/fission/fission/pkg/fission-cli/cmd/package/util" "github.com/fission/fission/pkg/fission-cli/cmd/spec" spectypes "github.com/fission/fission/pkg/fission-cli/cmd/spec/types" + "github.com/fission/fission/pkg/fission-cli/console" + flagkey "github.com/fission/fission/pkg/fission-cli/flag/key" "github.com/fission/fission/pkg/fission-cli/util" "github.com/fission/fission/pkg/utils" + uuid "github.com/satori/go.uuid" ) // CreateArchive returns a fv1.Archive made from an archive . If specFile, then // create an archive upload spec in the specs directory; otherwise // upload the archive using client. noZip avoids zipping the // includeFiles, but is ignored if there's more than one includeFile. -func CreateArchive(client *client.Client, includeFiles []string, noZip bool, specDir string, specFile string) (*fv1.Archive, error) { +func CreateArchive(client *client.Client, includeFiles []string, noZip bool, insecure bool, checksum string, specDir string, specFile string) (*fv1.Archive, error) { // get root dir var rootDir string var err error @@ -96,51 +100,80 @@ func CreateArchive(client *client.Client, includeFiles []string, noZip bool, spe return nil, errs.ErrorOrNil() } - if len(specFile) > 0 { - if len(fileURL) > 0 { + if len(fileURL) > 0 { + if insecure { return &fv1.Archive{ Type: fv1.ArchiveTypeUrl, URL: fileURL, }, nil - } else { - // create an ArchiveUploadSpec and reference it from the archive - aus := &spectypes.ArchiveUploadSpec{ - Name: archiveName("", includeFiles), - IncludeGlobs: includeFiles, - } - - // check if this AUS exists in the specs; if so, don't create a new one - fr, err := spec.ReadSpecs(specDir) - if err != nil { - return nil, errors.Wrap(err, "error reading specs") - } - - obj := fr.SpecExists(aus, true, true) - if obj != nil { - oldAus := obj.(*spectypes.ArchiveUploadSpec) - fmt.Printf("Re-using previously created archive %v\n", oldAus.Name) - aus.Name = oldAus.Name - } else { - // save the uploadspec - err := spec.SpecSave(*aus, specFile) - if err != nil { - return nil, errors.Wrapf(err, "write spec file %v", specFile) - } - } - // create the archive object - archive := fv1.Archive{ - Type: fv1.ArchiveTypeUrl, - URL: fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name), - } - return &archive, nil } + + var csum *fv1.Checksum + + if len(checksum) > 0 { + csum = &fv1.Checksum{ + Type: fv1.ChecksumTypeSHA256, + Sum: checksum, + } + } else { + console.Info(fmt.Sprintf("Downloading file to generate SHA256 checksum. To skip this step, please use --%v / --%v / --%v", + flagkey.PkgSrcChecksum, flagkey.PkgDeployChecksum, flagkey.PkgInsecure)) + + tmpDir, err := utils.GetTempDir() + if err != nil { + return nil, err + } + + file := filepath.Join(tmpDir, uuid.NewV4().String()) + err = utils.DownloadUrl(context.Background(), http.DefaultClient, fileURL, file) + if err != nil { + return nil, errors.Wrap(err, "error downloading file from the given URL") + } + + csum, err = utils.GetFileChecksum(file) + if err != nil { + return nil, errors.Wrap(err, "error generating file SHA256 checksum") + } + } + + return &fv1.Archive{ + Type: fv1.ArchiveTypeUrl, + URL: fileURL, + Checksum: *csum, + }, nil } - if len(fileURL) > 0 { - return &fv1.Archive{ + if len(specFile) > 0 { + // create an ArchiveUploadSpec and reference it from the archive + aus := &spectypes.ArchiveUploadSpec{ + Name: archiveName("", includeFiles), + IncludeGlobs: includeFiles, + } + + // check if this AUS exists in the specs; if so, don't create a new one + fr, err := spec.ReadSpecs(specDir) + if err != nil { + return nil, errors.Wrap(err, "error reading specs") + } + + obj := fr.SpecExists(aus, true, true) + if obj != nil { + oldAus := obj.(*spectypes.ArchiveUploadSpec) + fmt.Printf("Re-using previously created archive %v\n", oldAus.Name) + aus.Name = oldAus.Name + } else { + // save the uploadspec + err := spec.SpecSave(*aus, specFile) + if err != nil { + return nil, errors.Wrapf(err, "write spec file %v", specFile) + } + } + // create the archive object + archive := fv1.Archive{ Type: fv1.ArchiveTypeUrl, - URL: fileURL, - }, nil + URL: fmt.Sprintf("%v%v", spec.ARCHIVE_URL_PREFIX, aus.Name), + } + return &archive, nil } archivePath, err := makeArchiveFile("", includeFiles, noZip) diff --git a/pkg/fission-cli/cmd/package/update.go b/pkg/fission-cli/cmd/package/update.go index 8c6df24e..96bcf781 100644 --- a/pkg/fission-cli/cmd/package/update.go +++ b/pkg/fission-cli/cmd/package/update.go @@ -20,6 +20,7 @@ import ( "fmt" "time" + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,15 +32,10 @@ import ( ) type UpdateSubCommand struct { - client *client.Client - pkgName string - pkgNamespace string - force bool - envName string - envNamespace string - srcArchiveFiles []string - deployArchiveFiles []string - buildcmd string + client *client.Client + pkgName string + pkgNamespace string + force bool } func Update(input cli.Input) error { @@ -65,11 +61,6 @@ func (opts *UpdateSubCommand) complete(input cli.Input) error { opts.pkgName = input.String(flagkey.PkgName) opts.pkgNamespace = input.String(flagkey.NamespacePackage) opts.force = input.Bool(flagkey.PkgForce) - opts.envName = input.String(flagkey.PkgEnvironment) - opts.envNamespace = input.String(flagkey.NamespaceEnvironment) - opts.srcArchiveFiles = input.StringSlice(flagkey.PkgSrcArchive) - opts.deployArchiveFiles = input.StringSlice(flagkey.PkgDeployArchive) - opts.buildcmd = input.String(flagkey.PkgBuildCmd) return nil } @@ -82,88 +73,111 @@ func (opts *UpdateSubCommand) run(input cli.Input) error { return errors.Wrap(err, "get package") } - // if the new env specified is the same as the old one, no need to update package - // same is true for all update parameters, but, for now, we dont check all of them - because, its ok to - // re-write the object with same old values, we just end up getting a new resource version for the object. - if len(opts.envName) > 0 && opts.envName == pkg.Spec.Environment.Name { - opts.envName = "" - } - - if opts.envNamespace == pkg.Spec.Environment.Namespace { - opts.envNamespace = "" - } + forceUpdate := input.Bool(flagkey.PkgForce) fnList, err := GetFunctionsByPackage(opts.client, pkg.Metadata.Name, pkg.Metadata.Namespace) if err != nil { - return errors.Wrap(err, "get function list") + return errors.Wrap(err, "error getting function list") } - if !opts.force && len(fnList) > 1 { - return errors.New("Package is used by multiple functions, use --force to force update") + if !forceUpdate && len(fnList) > 1 { + return errors.Errorf("package is used by multiple functions, use --%v to force update", flagkey.PkgForce) } - newPkgMeta, err := UpdatePackage(opts.client, pkg, - opts.envName, opts.envNamespace, opts.srcArchiveFiles, - opts.deployArchiveFiles, opts.buildcmd, false, false) + newPkgMeta, err := UpdatePackage(input, opts.client, pkg) if err != nil { - return errors.Wrap(err, "update package") + return errors.Wrap(err, "error updating package") } - // update resource version of package reference of functions that shared the same package - for _, fn := range fnList { - fn.Spec.Package.PackageRef.ResourceVersion = newPkgMeta.ResourceVersion - _, err := opts.client.FunctionUpdate(&fn) + if pkg.Metadata.ResourceVersion != newPkgMeta.ResourceVersion { + err = UpdateFunctionPackageResourceVersion(opts.client, newPkgMeta, fnList...) if err != nil { - return errors.Wrap(err, "update function") + return errors.Wrap(err, "error updating function package reference resource version") } } - fmt.Printf("Package '%v' updated\n", newPkgMeta.GetName()) return nil } -func UpdatePackage(client *client.Client, pkg *fv1.Package, envName, envNamespace string, - srcArchiveFiles []string, deployArchiveFiles []string, buildcmd string, forceRebuild bool, noZip bool) (*metav1.ObjectMeta, error) { +func UpdatePackage(input cli.Input, client *client.Client, pkg *fv1.Package) (*metav1.ObjectMeta, error) { + envName := input.String(flagkey.PkgEnvironment) + envNamespace := input.String(flagkey.NamespaceEnvironment) + srcArchiveFiles := input.StringSlice(flagkey.PkgSrcArchive) + deployArchiveFiles := input.StringSlice(flagkey.PkgDeployArchive) + buildcmd := input.String(flagkey.PkgBuildCmd) + insecure := input.Bool(flagkey.PkgInsecure) + deployChecksum := input.String(flagkey.PkgDeployChecksum) + srcChecksum := input.String(flagkey.PkgSrcChecksum) + code := input.String(flagkey.PkgCode) - needToBuild := false + noZip := false + needToRebuild := false + needToUpdate := false - if len(envName) > 0 { + if input.IsSet(flagkey.PkgCode) { + deployArchiveFiles = append(deployArchiveFiles, code) + noZip = true + needToUpdate = true + } + + if input.IsSet(flagkey.PkgEnvironment) { pkg.Spec.Environment.Name = envName - needToBuild = true + needToRebuild = true + needToUpdate = true } - if len(envNamespace) > 0 { + if input.IsSet(flagkey.NamespaceEnvironment) { pkg.Spec.Environment.Namespace = envNamespace - needToBuild = true + needToRebuild = true + needToUpdate = true } - if len(buildcmd) > 0 { + if input.IsSet(flagkey.PkgBuildCmd) { pkg.Spec.BuildCommand = buildcmd - needToBuild = true + needToRebuild = true + needToUpdate = true } - if len(srcArchiveFiles) > 0 { - srcArchive, err := CreateArchive(client, srcArchiveFiles, false, "", "") + if input.IsSet(flagkey.PkgSrcArchive) { + srcArchive, err := CreateArchive(client, srcArchiveFiles, noZip, insecure, srcChecksum, "", "") if err != nil { - return nil, err + return nil, errors.Wrap(err, "error creating source archive") } pkg.Spec.Source = *srcArchive - needToBuild = true + needToRebuild = true + needToUpdate = true + } else if input.IsSet(flagkey.PkgSrcChecksum) { + pkg.Spec.Source.Checksum = fv1.Checksum{ + Type: fv1.ChecksumTypeSHA256, + Sum: srcChecksum, + } + needToUpdate = true } - if len(deployArchiveFiles) > 0 { - deployArchive, err := CreateArchive(client, deployArchiveFiles, noZip, "", "") + if input.IsSet(flagkey.PkgDeployArchive) || input.IsSet(flagkey.PkgCode) { + deployArchive, err := CreateArchive(client, deployArchiveFiles, noZip, insecure, deployChecksum, "", "") if err != nil { - return nil, err + return nil, errors.Wrap(err, "error creating deploy archive") } pkg.Spec.Deployment = *deployArchive // Users may update the env, envNS and deploy archive at the same time, // but without the source archive. In this case, we should set needToBuild to false - needToBuild = false + needToRebuild = false + needToUpdate = true + } else if input.IsSet(flagkey.PkgDeployChecksum) { + pkg.Spec.Deployment.Checksum = fv1.Checksum{ + Type: fv1.ChecksumTypeSHA256, + Sum: srcChecksum, + } + needToUpdate = true + } + + if !needToUpdate { + return &pkg.Metadata, nil } // Set package as pending status when needToBuild is true - if needToBuild || forceRebuild { + if needToRebuild { // change into pending state to trigger package build pkg.Status = fv1.PackageStatus{ BuildStatus: fv1.BuildStatusPending, @@ -176,9 +190,26 @@ func UpdatePackage(client *client.Client, pkg *fv1.Package, envName, envNamespac return nil, errors.Wrap(err, "update package") } + fmt.Printf("Package '%v' updated\n", newPkgMeta.GetName()) + return newPkgMeta, err } +func UpdateFunctionPackageResourceVersion(client *client.Client, pkgMeta *metav1.ObjectMeta, fnList ...fv1.Function) error { + errs := &multierror.Error{} + + // update resource version of package reference of functions that shared the same package + for _, fn := range fnList { + fn.Spec.Package.PackageRef.ResourceVersion = pkgMeta.ResourceVersion + _, err := client.FunctionUpdate(&fn) + if err != nil { + errs = multierror.Append(errs, errors.Wrapf(err, "error updating package resource version of function '%v'", fn.Metadata.Name)) + } + } + + return errs.ErrorOrNil() +} + func updatePackageStatus(client *client.Client, pkg *fv1.Package, status fv1.BuildStatus) (*metav1.ObjectMeta, error) { switch status { case fv1.BuildStatusNone, fv1.BuildStatusPending, fv1.BuildStatusRunning, fv1.BuildStatusSucceeded, fv1.CanaryConfigStatusAborted: diff --git a/pkg/fission-cli/flag/flag.go b/pkg/fission-cli/flag/flag.go index 5f20d6ab..de8ffd94 100644 --- a/pkg/fission-cli/flag/flag.go +++ b/pkg/fission-cli/flag/flag.go @@ -150,16 +150,19 @@ var ( KwObjType = Flag{Type: String, Name: flagkey.KwObjType, Usage: "Type of resource to watch (Pod, Service, etc.)", DefaultValue: "pod"} KwLabels = Flag{Type: String, Name: flagkey.KwLabels, Usage: "Label selector of the form a=b,c=d"} - PkgName = Flag{Type: String, Name: flagkey.PkgName, Usage: "Package name"} - PkgForce = Flag{Type: Bool, Name: flagkey.PkgForce, Short: "f", Usage: "Force update a package even if it is used by one or more functions"} - PkgEnvironment = Flag{Type: String, Name: flagkey.PkgEnvironment, Usage: "Environment name"} - PkgBuildCmd = Flag{Type: String, Name: flagkey.PkgBuildCmd, Usage: "Build command for builder to run with"} - PkgOutput = Flag{Type: String, Name: flagkey.PkgOutput, Short: "o", Usage: "Output filename to save archive content"} - PkgStatus = Flag{Type: String, Name: flagkey.PkgStatus, Usage: `Filter packages by status`} - PkgOrphan = Flag{Type: Bool, Name: flagkey.PkgOrphan, Usage: "Orphan packages that are not referenced by any function"} - PkgCode = Flag{Type: String, Name: flagkey.PkgCode, Usage: "URL or local path for single file source code"} - PkgDeployArchive = Flag{Type: StringSlice, Name: flagkey.PkgDeployArchive, Aliases: []string{"deploy"}, Usage: "URL or local paths for binary archive"} - PkgSrcArchive = Flag{Type: StringSlice, Name: flagkey.PkgSrcArchive, Aliases: []string{"source", "src"}, Usage: "URL or local paths for source archive"} + PkgName = Flag{Type: String, Name: flagkey.PkgName, Usage: "Package name"} + PkgForce = Flag{Type: Bool, Name: flagkey.PkgForce, Short: "f", Usage: "Force update a package even if it is used by one or more functions"} + PkgEnvironment = Flag{Type: String, Name: flagkey.PkgEnvironment, Usage: "Environment name"} + PkgBuildCmd = Flag{Type: String, Name: flagkey.PkgBuildCmd, Usage: "Build command for builder to run with"} + PkgOutput = Flag{Type: String, Name: flagkey.PkgOutput, Short: "o", Usage: "Output filename to save archive content"} + PkgStatus = Flag{Type: String, Name: flagkey.PkgStatus, Usage: `Filter packages by status`} + PkgOrphan = Flag{Type: Bool, Name: flagkey.PkgOrphan, Usage: "Orphan packages that are not referenced by any function"} + PkgCode = Flag{Type: String, Name: flagkey.PkgCode, Usage: "URL or local path for single file source code"} + PkgDeployArchive = Flag{Type: StringSlice, Name: flagkey.PkgDeployArchive, Aliases: []string{"deploy"}, Usage: "URL or local paths for binary archive"} + PkgDeployChecksum = Flag{Type: String, Name: flagkey.PkgDeployChecksum, Usage: "SHA256 checksum of deploy archive when providing URL"} + PkgSrcArchive = Flag{Type: StringSlice, Name: flagkey.PkgSrcArchive, Aliases: []string{"source", "src"}, Usage: "URL or local paths for source archive"} + PkgSrcChecksum = Flag{Type: String, Name: flagkey.PkgSrcChecksum, Usage: "SHA256 checksum of source archive when providing URL"} + PkgInsecure = Flag{Type: Bool, Name: flagkey.PkgInsecure, Usage: "Skip generating SHA256 checksum for file integrity validation"} SpecSave = Flag{Type: Bool, Name: flagkey.SpecSave, Usage: "Save to the spec directory instead of creating on cluster"} SpecDir = Flag{Type: String, Name: flagkey.SpecDir, Usage: "Directory to store specs, defaults to ./specs"} diff --git a/pkg/fission-cli/flag/key/key.go b/pkg/fission-cli/flag/key/key.go index a74969b6..2c29aff0 100644 --- a/pkg/fission-cli/flag/key/key.go +++ b/pkg/fission-cli/flag/key/key.go @@ -104,16 +104,19 @@ const ( KwObjType = "type" KwLabels = "labels" - PkgName = resourceName - PkgForce = force - PkgEnvironment = "env" - PkgCode = "code" - PkgSrcArchive = "sourcearchive" - PkgDeployArchive = "deployarchive" - PkgBuildCmd = "buildcmd" - PkgOutput = Output - PkgStatus = "status" - PkgOrphan = "orphan" + PkgName = resourceName + PkgForce = force + PkgEnvironment = "env" + PkgCode = "code" + PkgSrcArchive = "sourcearchive" + PkgDeployArchive = "deployarchive" + PkgSrcChecksum = "srcchecksum" + PkgDeployChecksum = "deploychecksum" + PkgInsecure = "insecure" + PkgBuildCmd = "buildcmd" + PkgOutput = Output + PkgStatus = "status" + PkgOrphan = "orphan" SpecSave = "spec" SpecDir = "specdir" diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index d896032a..41fb5eee 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -17,12 +17,15 @@ limitations under the License. package utils import ( + "context" "crypto/sha256" "encoding/hex" "fmt" + "golang.org/x/net/context/ctxhttp" "io" "io/ioutil" "net" + "net/http" "os" "path/filepath" "strings" @@ -198,3 +201,35 @@ func GetChecksum(src io.Reader) (*fv1.Checksum, error) { func IsURL(str string) bool { return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") } + +func DownloadUrl(ctx context.Context, httpClient *http.Client, url string, localPath string) error { + resp, err := ctxhttp.Get(ctx, httpClient, url) + if err != nil { + return err + } + defer resp.Body.Close() + + w, err := os.Create(localPath) + if err != nil { + return err + } + defer w.Close() + + _, err = io.Copy(w, resp.Body) + if err != nil { + return err + } + + // flushing write buffer to file + err = w.Sync() + if err != nil { + return err + } + + err = os.Chmod(localPath, 0600) + if err != nil { + return err + } + + return nil +} diff --git a/test/test_utils.sh b/test/test_utils.sh index 256928c6..dbef4ff3 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -512,6 +512,7 @@ run_all_tests() { export JOBS=6 $ROOT/test/run_test.sh \ $ROOT/test/tests/test_canary.sh \ + $ROOT/test/tests/test_fn_update/test_idle_objects_reaper.sh \ $ROOT/test/tests/mqtrigger/kafka/test_kafka.sh \ $ROOT/test/tests/test_annotations.sh \ $ROOT/test/tests/test_archive_pruner.sh \ @@ -519,7 +520,6 @@ run_all_tests() { $ROOT/test/tests/test_buildermgr.sh \ $ROOT/test/tests/test_env_vars.sh \ $ROOT/test/tests/test_environments/test_python_env.sh \ - $ROOT/test/tests/test_fn_update/test_idle_objects_reaper.sh \ $ROOT/test/tests/test_function_test/test_fn_test.sh \ $ROOT/test/tests/test_function_update.sh \ $ROOT/test/tests/test_ingress.sh \ @@ -527,6 +527,7 @@ run_all_tests() { $ROOT/test/tests/test_logging/test_function_logs.sh \ $ROOT/test/tests/test_node_hello_http.sh \ $ROOT/test/tests/test_package_command.sh \ + $ROOT/test/tests/test_package_checksum.sh \ $ROOT/test/tests/test_pass.sh \ $ROOT/test/tests/test_router_cache_invalidation.sh \ $ROOT/test/tests/test_specs/test_spec.sh \ @@ -539,7 +540,7 @@ run_all_tests() { FAILURES=$? # FIXME: run tests with newdeploy one by one. - export JOBS=2 + export JOBS=3 $ROOT/test/run_test.sh \ $ROOT/test/tests/test_backend_newdeploy.sh \ $ROOT/test/tests/test_environments/test_java_builder.sh \ diff --git a/test/tests/test_package_checksum.sh b/test/tests/test_package_checksum.sh new file mode 100755 index 00000000..cbfbac33 --- /dev/null +++ b/test/tests/test_package_checksum.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +set -euo pipefail +source $(dirname $0)/../utils.sh + +TEST_ID=$(generate_test_id) +echo "TEST_ID = $TEST_ID" + +tmp_dir="/tmp/test-$TEST_ID" +mkdir -p $tmp_dir + +ROOT=$(dirname $0)/../.. + +checkpkgsum() { + local pkg=$1 + local sum=$2 + + pkgsum=$(kubectl -n default get packages ${pkg} -o jsonpath='{.spec.deployment.checksum.sum}') + + if [ "${sum}" != "${pkgsum}" ]; then + log "have different sha256 checksum: ${sum} vs. ${pkgsum}" + kubectl -n default get packages ${pkg} -o yaml + exit 1 + fi +} + +cleanup() { + log "Cleaning up..." + clean_resource_by_id $TEST_ID + rm -rf $tmp_dir +} + +if [ -z "${TEST_NOCLEANUP:-}" ]; then + trap cleanup EXIT +else + log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards." +fi + +log "Download test script" + +url1="https://raw.githubusercontent.com/fission/fission/master/examples/nodejs/hello.js" +url2="https://raw.githubusercontent.com/fission/fission/master/examples/nodejs/hello-callback.js" + +wget ${url1} +sum=$(shasum -a 256 hello.js|cut -d' ' -f 1) + +wget ${url2} +sum2=$(shasum -a 256 hello-callback.js|cut -d' ' -f 1) + +env="nodejs-${TEST_ID}" + +log "Function with file URL" +fn1="fn1-$TEST_ID" +fission env create --name ${env} --image $NODE_RUNTIME_IMAGE --period 5 +fission fn create --name ${fn1} --env ${env} --code ${url1} +pkgname=$(kubectl -n default get functions ${fn1} -o jsonpath="{.spec.package.packageref.name}") +checkpkgsum ${pkgname} ${sum} + +log "Creating route" +fission route create --name ${fn1} --function ${fn1} --url /${fn1} --method GET +sleep 3 + +timeout 60 bash -c "test_fn ${fn1} 'hello, world'" + +log "Update function with file URL" +fission fn update --name ${fn1} --env ${env} --code ${url2} +pkgname=$(kubectl -n default get functions ${fn1} -o jsonpath="{.spec.package.packageref.name}") +checkpkgsum ${pkgname} ${sum2} + +sleep 3 + +timeout 60 bash -c "test_fn ${fn1} 'Hello, world callback!'" + +log "Function with file URL & checksum" +fn2="fn2-$TEST_ID" +fission fn create --name ${fn2} --env ${env} --code ${url1} --deploychecksum ${sum} +pkgname=$(kubectl -n default get functions ${fn2} -o jsonpath="{.spec.package.packageref.name}") +checkpkgsum ${pkgname} ${sum} + +log "Creating route" +fission route create --name ${fn2} --function ${fn2} --url /${fn2} --method GET + +timeout 60 bash -c "test_fn ${fn2} 'hello, world'" + +log "Function with file URL & insecure" +fn3="fn3-$TEST_ID" +fission fn create --name ${fn3} --env ${env} --code ${url1} --insecure +pkgname=$(kubectl -n default get functions ${fn3} -o jsonpath="{.spec.package.packageref.name}") +checkpkgsum ${pkgname} "" + +log "Creating route" +fission route create --name ${fn3} --function ${fn3} --url /${fn3} --method GET +sleep 3 + +timeout 60 bash -c "test_fn ${fn3} 'hello, world'" + +pkg1="pkg1-$TEST_ID" +pkg2="pkg2-$TEST_ID" +pkg3="pkg3-$TEST_ID" + +fission pkg create --name ${pkg1} --env ${env} --code ${url1} +checkpkgsum ${pkg1} ${sum} +fission pkg update --name ${pkg1} --env ${env} --code ${url2} +checkpkgsum ${pkg1} ${sum2} +fission pkg create --name ${pkg2} --env ${env} --code ${url1} --deploychecksum ${sum} +checkpkgsum ${pkg2} ${sum} +fission pkg create --name ${pkg3} --env ${env} --code ${url1} --insecure +checkpkgsum ${pkg3} "" + +log "Test PASSED" + +exit 0 diff --git a/test/utils.sh b/test/utils.sh index cb04b7cb..bf37501a 100755 --- a/test/utils.sh +++ b/test/utils.sh @@ -44,6 +44,10 @@ clean_resource_by_id() { } test_fn() { + if [ -z $FISSION_ROUTER ]; then + log "Environment FISSION_ROUTER not set" + exit 1 + fi url="http://$FISSION_ROUTER/$1" expect=$2 test_response $url $expect