Support annotations in environment specs (#733)

* Added annotations to runtime and builder environment specs
* Use crd annotations for env pod

buildermgr/envwatcher.go

@@ -498,9 +498,12 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns
 	sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
 	var replicas int32 = 1
 
-	podAnnotation := make(map[string]string)
+	podAnnotations := env.Metadata.Annotations
+	if podAnnotations == nil {
+		podAnnotations = make(map[string]string)
+	}
 	if envw.useIstio && env.Spec.AllowAccessToExternalNetwork {
-		podAnnotation["sidecar.istio.io/inject"] = "false"
+		podAnnotations["sidecar.istio.io/inject"] = "false"
 	}
 
 	deployment := &v1beta1.Deployment{
@@ -517,7 +520,7 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns
 			Template: apiv1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      sel,
-					Annotations: podAnnotation,
+					Annotations: podAnnotations,
 				},
 				Spec: apiv1.PodSpec{
 					Volumes: []apiv1.Volume{

executor/newdeploy/newdeploy.go

@@ -185,9 +185,12 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen
 		return nil, err
 	}
 
-	podAnnotation := make(map[string]string)
+	podAnnotations := env.Metadata.Annotations
+	if podAnnotations == nil {
+		podAnnotations = make(map[string]string)
+	}
 	if deploy.useIstio && env.Spec.AllowAccessToExternalNetwork {
-		podAnnotation["sidecar.istio.io/inject"] = "false"
+		podAnnotations["sidecar.istio.io/inject"] = "false"
 	}
 	resources := deploy.getResources(env, fn)
 
@@ -204,7 +207,7 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen
 			Template: apiv1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      deployLabels,
-					Annotations: podAnnotation,
+					Annotations: podAnnotations,
 				},
 				Spec: apiv1.PodSpec{
 					Volumes: []apiv1.Volume{

executor/poolmgr/gp.go

@@ -485,10 +485,12 @@ func (gp *GenericPool) createPool() error {
 		gracePeriodSeconds = gp.env.Spec.TerminationGracePeriod
 	}
 
-	podAnnotation := make(map[string]string)
+	podAnnotations := gp.env.Metadata.Annotations
-
+	if podAnnotations == nil {
+		podAnnotations = make(map[string]string)
+	}
 	if gp.useIstio && gp.env.Spec.AllowAccessToExternalNetwork {
-		podAnnotation["sidecar.istio.io/inject"] = "false"
+		podAnnotations["sidecar.istio.io/inject"] = "false"
 	}
 
 	deployment := &v1beta1.Deployment{
@@ -504,7 +506,7 @@ func (gp *GenericPool) createPool() error {
 			Template: apiv1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
 					Labels:      gp.labelsForPool,
-					Annotations: podAnnotation,
+					Annotations: podAnnotations,
 				},
 				Spec: apiv1.PodSpec{
 					Volumes: []apiv1.Volume{

test/tests/test_annotations.sh

@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# test_annotations.sh - tests whether a user is able to add pod annotations to a Fission environment deployment
+
+TEST_ID=$(date +%s)
+ENV=python-${TEST_ID}
+FN=foo-${TEST_ID}
+RESOURCE_NS=default # Change to test-specific namespace once we support namespaced CRDs
+FUNCTION_NS=${FUNCTION_NAMESPACE:-fission-function}
+BUILDER_NS=fission-builder
+LIST_ANNOTATIONS=go-template='{{range $key,$value := .metadata.annotations}}{{$key}}: {{$value}}{{"\n"}}{{end}}'
+
+# fs
+TEST_DIR=/tmp/${TEST_ID}
+ENV_SPEC_FILE=${TEST_DIR}/${ENV}.yaml
+FN_FILE=${TEST_DIR}/${FN}.yaml
+
+log_exec() {
+    cmd=$@
+    echo "> ${cmd}"
+    ${cmd}
+}
+
+cleanup() {
+    log "Cleaning up..."
+    kubectl -n ${RESOURCE_NS} delete environment/${ENV} || true
+    rm -rf ${TEST_DIR}
+
+}
+
+cleanup
+if [ -z "${TEST_NOCLEANUP:-}" ]; then
+    trap cleanup EXIT
+else
+    log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
+fi
+
+if ! stat ${TEST_DIR} >/dev/null 2>&1 ; then
+    mkdir ${TEST_DIR}
+fi
+
+getPodName() {
+    NS=$1
+    POD=$2
+    # find pod is ready to serve
+    JSONPATH="{range .items[*]}{'\n'}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}"
+    kubectl -n ${NS} get po -o jsonpath="$JSONPATH" \
+        | grep "Ready=True" \
+        | grep ${POD} \
+        | head -n 1 \
+        | cut -f1 -d":"
+}
+
+# retry function adapted from:
+# https://unix.stackexchange.com/questions/82598/how-do-i-write-a-retry-logic-in-script-to-keep-retrying-to-run-it-upto-5-times/82610
+function retry {
+  local n=1
+  local max=5
+  local delay=10 # pods take time to get ready
+  while true; do
+    "$@" && break || {
+      if [[ ${n} -lt ${max} ]]; then
+        ((n++))
+        echo "Command '$@' failed. Attempt $n/$max:"
+        sleep ${delay};
+      else
+        >&2 echo "The command has failed after $n attempts."
+        exit 1;
+      fi
+    }
+  done
+}
+
+# Deploy environment (using kubectl because the Fission cli does not support the container arguments)
+ANNOTATION_KEY="foo"
+ANNOTATION_VALUE="bar"
+echo "Writing environment config to $ENV_SPEC_FILE"
+cat > $ENV_SPEC_FILE <<- EOM
+apiVersion: fission.io/v1
+kind: Environment
+metadata:
+  name: ${ENV}
+  namespace: ${RESOURCE_NS}
+  annotations:
+    ${ANNOTATION_KEY}: ${ANNOTATION_VALUE}
+spec:
+  builder:
+    command: build
+    image: gcr.io/fission-ci/python-env-builder:test
+  runtime:
+    image: gcr.io/fission-ci/python-env:test
+  version: 2
+  poolsize: 1
+EOM
+log_exec kubectl -n ${RESOURCE_NS} apply -f ${ENV_SPEC_FILE}
+
+sleep 15
+# Wait for runtime and build env to be deployed
+retry getPodName ${FUNCTION_NS} ${ENV} | grep '.\+'
+runtimePod=$(getPodName ${FUNCTION_NS} ${ENV})
+echo "function pod: ${runtimePod}."
+retry getPodName ${BUILDER_NS} ${ENV} | grep '.\+'
+buildPod=$(getPodName ${BUILDER_NS} ${ENV})
+echo "builder pod: ${buildPod}."
+
+# Ensure pods are running/ready
+log "Waiting for ${FUNCTION_NS} ${ENV} to be available..."
+echo "> kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o \"${LIST_ANNOTATIONS}\""
+retry kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" > /dev/null
+log "Runtime pod ready."
+
+log "Waiting for ${BUILDER_NS} ${ENV} to be available..."
+echo "> kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o \"${LIST_ANNOTATIONS}\""
+retry kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" > /dev/null
+log "Builder pod ready."
+
+# Check if the annotation is set on the runtime pod
+status=0
+if kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" | grep "${ANNOTATION_KEY}: ${ANNOTATION_VALUE}"; then
+    log "Runtime annotation is correct."
+else
+    log "Runtime does not contain expected annotation: ${ANNOTATION_KEY}: ${ANNOTATION_VALUE}"
+    echo "--- Runtime Env ---"
+    kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" || true
+    echo "--- End Runtime Env ---"
+    status=5
+fi
+
+# Check if the annotation is set on the builder pod
+if kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" | grep "${ANNOTATION_KEY}: ${ANNOTATION_VALUE}" ; then
+    log "Builder annotation is correct."
+else
+    log "Builder does not contain expected annotation: ${ANNOTATION_KEY}: ${ANNOTATION_VALUE}"
+    echo "--- Builder Env ---"
+    kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" || true
+    echo "--- End Builder Env ---"
+    status=5
+fi
+exit ${status}