Fix spec shows source archive is not used (#1448)

pkg/fission-cli/cmd/spec/spec.go

@@ -297,35 +297,26 @@ func (fr *FissionResources) Validate(input cli.Input) error {
 	for _, p := range fr.Packages {
 		packages[MapKey(&p.Metadata)] = false
 
-		if strings.HasPrefix(p.Spec.Deployment.URL, ARCHIVE_URL_PREFIX) {
+		as := map[string]string{
-			// check archive refs from package
+			"source":     p.Spec.Source.URL,
-			aname := strings.TrimPrefix(p.Spec.Source.URL, ARCHIVE_URL_PREFIX)
+			"deployment": p.Spec.Deployment.URL,
-			if len(aname) > 0 {
-				if _, ok := archives[aname]; !ok {
-					result = multierror.Append(result, fmt.Errorf(
-						"%v: package '%v' references unknown source archive %v%v",
-						fr.SourceMap.Locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
-						p.Metadata.Name,
-						ARCHIVE_URL_PREFIX,
-						aname))
-				} else {
-					archives[aname] = true
-				}
-			}
 		}
 
-		if strings.HasPrefix(p.Spec.Deployment.URL, ARCHIVE_URL_PREFIX) {
+		for archiveType, u := range as {
-			aname := strings.TrimPrefix(p.Spec.Deployment.URL, ARCHIVE_URL_PREFIX)
+			if strings.HasPrefix(u, ARCHIVE_URL_PREFIX) {
-			if len(aname) > 0 {
+				aname := strings.TrimPrefix(u, ARCHIVE_URL_PREFIX)
-				if _, ok := archives[aname]; !ok {
+				if len(aname) > 0 {
-					result = multierror.Append(result, fmt.Errorf(
+					if _, ok := archives[aname]; !ok {
-						"%v: package '%v' references unknown deployment archive %v%v",
+						result = multierror.Append(result, fmt.Errorf(
-						fr.SourceMap.Locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
+							"%v: package '%v' references unknown %v archive '%v%v'",
-						p.Metadata.Name,
+							fr.SourceMap.Locations["Package"][p.Metadata.Namespace][p.Metadata.Name],
-						ARCHIVE_URL_PREFIX,
+							p.Metadata.Name,
-						aname))
+							archiveType,
-				} else {
+							ARCHIVE_URL_PREFIX,
-					archives[aname] = true
+							aname))
+					} else {
+						archives[aname] = true
+					}
 				}
 			}
 		}

test/test_utils.sh

@@ -534,6 +534,7 @@ run_all_tests() {
         $ROOT/test/tests/test_specs/test_spec_multifile.sh \
         $ROOT/test/tests/test_specs/test_ignore_hidden_file.sh \
         $ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh \
+        $ROOT/test/tests/test_specs/test_spec_archive/test_spec_archive.sh \
         $ROOT/test/tests/test_environments/test_tensorflow_serving_env.sh \
         $ROOT/test/tests/test_environments/test_go_env.sh \
         $ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \
@@ -541,7 +542,6 @@ run_all_tests() {
         $ROOT/test/tests/test_huge_response/test_huge_response.sh
     FAILURES=$?
 
-    # FIXME: run tests with newdeploy one by one.
     export JOBS=3
     $ROOT/test/run_test.sh \
         $ROOT/test/tests/test_backend_newdeploy.sh \

test/tests/test_specs/test_spec_archive/specs/README

@@ -0,0 +1,42 @@
+
+Fission Specs
+=============
+
+This is a set of specifications for a Fission app.  This includes functions,
+environments, and triggers; we collectively call these things "resources".
+
+How to use these specs
+----------------------
+
+These specs are handled with the 'fission spec' command.  See 'fission spec --help'.
+
+'fission spec apply' will "apply" all resources specified in this directory to your
+cluster.  That means it checks what resources exist on your cluster, what resources are
+specified in the specs directory, and reconciles the difference by creating, updating or
+deleting resources on the cluster.
+
+'fission spec apply' will also package up your source code (or compiled binaries) and
+upload the archives to the cluster if needed.  It uses 'ArchiveUploadSpec' resources in
+this directory to figure out which files to archive.
+
+You can use 'fission spec apply --watch' to watch for file changes and continuously keep
+the cluster updated.
+
+You can add YAMLs to this directory by writing them manually, but it's easier to generate
+them.  Use 'fission function create --spec' to generate a function spec,
+'fission environment create --spec' to generate an environment spec, and so on.
+
+You can edit any of the files in this directory, except 'fission-deployment-config.yaml',
+which contains a UID that you should never change.  To apply your changes simply use
+'fission spec apply'.
+
+fission-deployment-config.yaml
+------------------------------
+
+fission-deployment-config.yaml contains a UID.  This UID is what fission uses to correlate
+resources on the cluster to resources in this directory.
+
+All resources created by 'fission spec apply' are annotated with this UID.  Resources on
+the cluster that are _not_ annotated with this UID are never modified or deleted by
+fission.
+

test/tests/test_specs/test_spec_archive/specs/env-node.yaml

@@ -0,0 +1,16 @@
+apiVersion: fission.io/v1
+kind: Environment
+metadata:
+  creationTimestamp: null
+  name: dummyfoobarnode
+  namespace: default
+spec:
+  builder:
+    command: build
+    image: fission/node-builder:1.6.0
+  keeparchive: false
+  poolsize: 3
+  runtime:
+    image: fission/node-env:1.6.0
+  terminationGracePeriod: 20
+  version: 2

test/tests/test_specs/test_spec_archive/specs/fission-deployment-config.yaml

@@ -0,0 +1,7 @@
+# This file is generated by the 'fission spec init' command.
+# See the README in this directory for background and usage information.
+# Do not edit the UID below: that will break 'fission spec apply'
+apiVersion: fission.io/v1
+kind: DeploymentConfig
+name: test-spec-archive
+uid: 04b21526-8873-4dc2-b897-e87ed5347670

test/tests/test_specs/test_spec_archive/specs/functions.yaml

@@ -0,0 +1,56 @@
+apiVersion: fission.io/v1
+kind: Function
+metadata:
+  creationTimestamp: null
+  name: sourcearchive
+  namespace: default
+spec:
+  InvokeStrategy:
+    ExecutionStrategy:
+      ExecutorType: poolmgr
+      MaxScale: 0
+      MinScale: 0
+      SpecializationTimeout: 0
+      TargetCPUPercent: 0
+    StrategyType: execution
+  configmaps: null
+  environment:
+    name: dummyfoobarnode
+    namespace: default
+  functionTimeout: 60
+  package:
+    functionName: source
+    packageref:
+      name: sourcearchive
+      namespace: default
+  resources: {}
+  secrets: null
+
+---
+apiVersion: fission.io/v1
+kind: Function
+metadata:
+  creationTimestamp: null
+  name: deployarchive
+  namespace: default
+spec:
+  InvokeStrategy:
+    ExecutionStrategy:
+      ExecutorType: poolmgr
+      MaxScale: 0
+      MinScale: 0
+      SpecializationTimeout: 0
+      TargetCPUPercent: 0
+    StrategyType: execution
+  configmaps: null
+  environment:
+    name: dummyfoobarnode
+    namespace: default
+  functionTimeout: 60
+  package:
+    functionName: deploy
+    packageref:
+      name: deployarchive
+      namespace: default
+  resources: {}
+  secrets: null

test/tests/test_specs/test_spec_archive/specs/pkg-document-deploy-archive.yaml

@@ -0,0 +1,24 @@
+include:
+  - func/*
+kind: ArchiveUploadSpec
+name: functions-deploy-archive
+
+---
+apiVersion: fission.io/v1
+kind: Package
+metadata:
+  creationTimestamp: null
+  name: deployarchive
+  namespace: default
+spec:
+  deployment:
+    checksum: {}
+    type: url
+    url: archive://functions-source-archive
+  environment:
+    name: dummyfoobarnode
+    namespace: default
+  source:
+    checksum: {}
+status:
+  buildstatus: none

test/tests/test_specs/test_spec_archive/specs/pkg-document-source-archive.yaml

@@ -0,0 +1,24 @@
+include:
+  - func/*
+kind: ArchiveUploadSpec
+name: functions-source-archive
+
+---
+apiVersion: fission.io/v1
+kind: Package
+metadata:
+  creationTimestamp: null
+  name: sourcearchive
+  namespace: default
+spec:
+  deployment:
+    checksum: {}
+  environment:
+    name: dummyfoobarnode
+    namespace: default
+  source:
+    checksum: {}
+    type: url
+    url: archive://functions-deploy-archive
+status:
+  buildstatus: pending

test/tests/test_specs/test_spec_archive/test_spec_archive.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -euo pipefail
+source $(dirname $0)/../../../utils.sh
+ROOT=` realpath $(dirname $0)/../../../../`
+
+cleanup() {
+    log "Cleaning up..."
+    fission spec destroy
+    rm -rf func
+    popd
+}
+
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+pushd $(dirname $0)
+
+[ -d specs ]
+[ -f specs/README ]
+[ -f specs/fission-deployment-config.yaml ]
+
+mkdir -p func
+cp $ROOT/examples/nodejs/hello.js func/deploy.js
+cp $ROOT/examples/nodejs/hello.js func/source.js
+
+fission spec destroy || true
+
+log "Apply specs"
+fission --verbosity 2 spec apply
+
+log "verify deployarchive function works"
+fission fn test --name deployarchive
+
+timeout 60s bash -c "waitBuild sourcearchive"
+
+log "verify sourcearchive function works"
+fission fn test --name sourcearchive
+
+log "Test PASSED"