From 032d1a8b9a49045033bebd18ece83a569cdb3fb4 Mon Sep 17 00:00:00 2001 From: Erwin van Eyk Date: Fri, 29 Jun 2018 23:38:22 +0200 Subject: [PATCH] Support annotations in environment specs (#733) * Added annotations to runtime and builder environment specs * Use crd annotations for env pod --- buildermgr/envwatcher.go | 9 +- executor/newdeploy/newdeploy.go | 9 +- executor/poolmgr/gp.go | 10 ++- test/tests/test_annotations.sh | 141 ++++++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 10 deletions(-) create mode 100755 test/tests/test_annotations.sh diff --git a/buildermgr/envwatcher.go b/buildermgr/envwatcher.go index 22432592..93ad0f65 100644 --- a/buildermgr/envwatcher.go +++ b/buildermgr/envwatcher.go @@ -498,9 +498,12 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion) var replicas int32 = 1 - podAnnotation := make(map[string]string) + podAnnotations := env.Metadata.Annotations + if podAnnotations == nil { + podAnnotations = make(map[string]string) + } if envw.useIstio && env.Spec.AllowAccessToExternalNetwork { - podAnnotation["sidecar.istio.io/inject"] = "false" + podAnnotations["sidecar.istio.io/inject"] = "false" } deployment := &v1beta1.Deployment{ @@ -517,7 +520,7 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns Template: apiv1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: sel, - Annotations: podAnnotation, + Annotations: podAnnotations, }, Spec: apiv1.PodSpec{ Volumes: []apiv1.Volume{ diff --git a/executor/newdeploy/newdeploy.go b/executor/newdeploy/newdeploy.go index 2de9ce3e..1387c0fe 100644 --- a/executor/newdeploy/newdeploy.go +++ b/executor/newdeploy/newdeploy.go @@ -185,9 +185,12 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen return nil, err } - podAnnotation := make(map[string]string) + podAnnotations := env.Metadata.Annotations + if podAnnotations == nil { + podAnnotations = make(map[string]string) + } if deploy.useIstio && env.Spec.AllowAccessToExternalNetwork { - podAnnotation["sidecar.istio.io/inject"] = "false" + podAnnotations["sidecar.istio.io/inject"] = "false" } resources := deploy.getResources(env, fn) @@ -204,7 +207,7 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen Template: apiv1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: deployLabels, - Annotations: podAnnotation, + Annotations: podAnnotations, }, Spec: apiv1.PodSpec{ Volumes: []apiv1.Volume{ diff --git a/executor/poolmgr/gp.go b/executor/poolmgr/gp.go index 86c5ebea..21f78c2a 100644 --- a/executor/poolmgr/gp.go +++ b/executor/poolmgr/gp.go @@ -485,10 +485,12 @@ func (gp *GenericPool) createPool() error { gracePeriodSeconds = gp.env.Spec.TerminationGracePeriod } - podAnnotation := make(map[string]string) - + podAnnotations := gp.env.Metadata.Annotations + if podAnnotations == nil { + podAnnotations = make(map[string]string) + } if gp.useIstio && gp.env.Spec.AllowAccessToExternalNetwork { - podAnnotation["sidecar.istio.io/inject"] = "false" + podAnnotations["sidecar.istio.io/inject"] = "false" } deployment := &v1beta1.Deployment{ @@ -504,7 +506,7 @@ func (gp *GenericPool) createPool() error { Template: apiv1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: gp.labelsForPool, - Annotations: podAnnotation, + Annotations: podAnnotations, }, Spec: apiv1.PodSpec{ Volumes: []apiv1.Volume{ diff --git a/test/tests/test_annotations.sh b/test/tests/test_annotations.sh new file mode 100755 index 00000000..e42ea813 --- /dev/null +++ b/test/tests/test_annotations.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# test_annotations.sh - tests whether a user is able to add pod annotations to a Fission environment deployment + +TEST_ID=$(date +%s) +ENV=python-${TEST_ID} +FN=foo-${TEST_ID} +RESOURCE_NS=default # Change to test-specific namespace once we support namespaced CRDs +FUNCTION_NS=${FUNCTION_NAMESPACE:-fission-function} +BUILDER_NS=fission-builder +LIST_ANNOTATIONS=go-template='{{range $key,$value := .metadata.annotations}}{{$key}}: {{$value}}{{"\n"}}{{end}}' + +# fs +TEST_DIR=/tmp/${TEST_ID} +ENV_SPEC_FILE=${TEST_DIR}/${ENV}.yaml +FN_FILE=${TEST_DIR}/${FN}.yaml + +log_exec() { + cmd=$@ + echo "> ${cmd}" + ${cmd} +} + +cleanup() { + log "Cleaning up..." + kubectl -n ${RESOURCE_NS} delete environment/${ENV} || true + rm -rf ${TEST_DIR} + +} + +cleanup +if [ -z "${TEST_NOCLEANUP:-}" ]; then + trap cleanup EXIT +else + log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards." +fi + +if ! stat ${TEST_DIR} >/dev/null 2>&1 ; then + mkdir ${TEST_DIR} +fi + +getPodName() { + NS=$1 + POD=$2 + # find pod is ready to serve + JSONPATH="{range .items[*]}{'\n'}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}" + kubectl -n ${NS} get po -o jsonpath="$JSONPATH" \ + | grep "Ready=True" \ + | grep ${POD} \ + | head -n 1 \ + | cut -f1 -d":" +} + +# retry function adapted from: +# https://unix.stackexchange.com/questions/82598/how-do-i-write-a-retry-logic-in-script-to-keep-retrying-to-run-it-upto-5-times/82610 +function retry { + local n=1 + local max=5 + local delay=10 # pods take time to get ready + while true; do + "$@" && break || { + if [[ ${n} -lt ${max} ]]; then + ((n++)) + echo "Command '$@' failed. Attempt $n/$max:" + sleep ${delay}; + else + >&2 echo "The command has failed after $n attempts." + exit 1; + fi + } + done +} + +# Deploy environment (using kubectl because the Fission cli does not support the container arguments) +ANNOTATION_KEY="foo" +ANNOTATION_VALUE="bar" +echo "Writing environment config to $ENV_SPEC_FILE" +cat > $ENV_SPEC_FILE <<- EOM +apiVersion: fission.io/v1 +kind: Environment +metadata: + name: ${ENV} + namespace: ${RESOURCE_NS} + annotations: + ${ANNOTATION_KEY}: ${ANNOTATION_VALUE} +spec: + builder: + command: build + image: gcr.io/fission-ci/python-env-builder:test + runtime: + image: gcr.io/fission-ci/python-env:test + version: 2 + poolsize: 1 +EOM +log_exec kubectl -n ${RESOURCE_NS} apply -f ${ENV_SPEC_FILE} + +sleep 15 +# Wait for runtime and build env to be deployed +retry getPodName ${FUNCTION_NS} ${ENV} | grep '.\+' +runtimePod=$(getPodName ${FUNCTION_NS} ${ENV}) +echo "function pod: ${runtimePod}." +retry getPodName ${BUILDER_NS} ${ENV} | grep '.\+' +buildPod=$(getPodName ${BUILDER_NS} ${ENV}) +echo "builder pod: ${buildPod}." + +# Ensure pods are running/ready +log "Waiting for ${FUNCTION_NS} ${ENV} to be available..." +echo "> kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o \"${LIST_ANNOTATIONS}\"" +retry kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" > /dev/null +log "Runtime pod ready." + +log "Waiting for ${BUILDER_NS} ${ENV} to be available..." +echo "> kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o \"${LIST_ANNOTATIONS}\"" +retry kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" > /dev/null +log "Builder pod ready." + +# Check if the annotation is set on the runtime pod +status=0 +if kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" | grep "${ANNOTATION_KEY}: ${ANNOTATION_VALUE}"; then + log "Runtime annotation is correct." +else + log "Runtime does not contain expected annotation: ${ANNOTATION_KEY}: ${ANNOTATION_VALUE}" + echo "--- Runtime Env ---" + kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" || true + echo "--- End Runtime Env ---" + status=5 +fi + +# Check if the annotation is set on the builder pod +if kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" | grep "${ANNOTATION_KEY}: ${ANNOTATION_VALUE}" ; then + log "Builder annotation is correct." +else + log "Builder does not contain expected annotation: ${ANNOTATION_KEY}: ${ANNOTATION_VALUE}" + echo "--- Builder Env ---" + kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" || true + echo "--- End Builder Env ---" + status=5 +fi +exit ${status}