Support annotations in environment specs (#733)
* Added annotations to runtime and builder environment specs * Use crd annotations for env pod
This commit is contained in:
Erwin van Eyk
committed by
Ta-Ching Chen
Ta-Ching Chen
parent
65fd1d9fbb
commit
032d1a8b9a
@@ -498,9 +498,12 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns
|
||||
sel := envw.getLabels(env.Metadata.Name, ns, env.Metadata.ResourceVersion)
|
||||
var replicas int32 = 1
|
||||
|
||||
podAnnotation := make(map[string]string)
|
||||
podAnnotations := env.Metadata.Annotations
|
||||
if podAnnotations == nil {
|
||||
podAnnotations = make(map[string]string)
|
||||
}
|
||||
if envw.useIstio && env.Spec.AllowAccessToExternalNetwork {
|
||||
podAnnotation["sidecar.istio.io/inject"] = "false"
|
||||
podAnnotations["sidecar.istio.io/inject"] = "false"
|
||||
}
|
||||
|
||||
deployment := &v1beta1.Deployment{
|
||||
@@ -517,7 +520,7 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns
|
||||
Template: apiv1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: sel,
|
||||
Annotations: podAnnotation,
|
||||
Annotations: podAnnotations,
|
||||
},
|
||||
Spec: apiv1.PodSpec{
|
||||
Volumes: []apiv1.Volume{
|
||||
|
||||
@@ -185,9 +185,12 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen
|
||||
return nil, err
|
||||
}
|
||||
|
||||
podAnnotation := make(map[string]string)
|
||||
podAnnotations := env.Metadata.Annotations
|
||||
if podAnnotations == nil {
|
||||
podAnnotations = make(map[string]string)
|
||||
}
|
||||
if deploy.useIstio && env.Spec.AllowAccessToExternalNetwork {
|
||||
podAnnotation["sidecar.istio.io/inject"] = "false"
|
||||
podAnnotations["sidecar.istio.io/inject"] = "false"
|
||||
}
|
||||
resources := deploy.getResources(env, fn)
|
||||
|
||||
@@ -204,7 +207,7 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen
|
||||
Template: apiv1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: deployLabels,
|
||||
Annotations: podAnnotation,
|
||||
Annotations: podAnnotations,
|
||||
},
|
||||
Spec: apiv1.PodSpec{
|
||||
Volumes: []apiv1.Volume{
|
||||
|
||||
@@ -485,10 +485,12 @@ func (gp *GenericPool) createPool() error {
|
||||
gracePeriodSeconds = gp.env.Spec.TerminationGracePeriod
|
||||
}
|
||||
|
||||
podAnnotation := make(map[string]string)
|
||||
|
||||
podAnnotations := gp.env.Metadata.Annotations
|
||||
if podAnnotations == nil {
|
||||
podAnnotations = make(map[string]string)
|
||||
}
|
||||
if gp.useIstio && gp.env.Spec.AllowAccessToExternalNetwork {
|
||||
podAnnotation["sidecar.istio.io/inject"] = "false"
|
||||
podAnnotations["sidecar.istio.io/inject"] = "false"
|
||||
}
|
||||
|
||||
deployment := &v1beta1.Deployment{
|
||||
@@ -504,7 +506,7 @@ func (gp *GenericPool) createPool() error {
|
||||
Template: apiv1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: gp.labelsForPool,
|
||||
Annotations: podAnnotation,
|
||||
Annotations: podAnnotations,
|
||||
},
|
||||
Spec: apiv1.PodSpec{
|
||||
Volumes: []apiv1.Volume{
|
||||
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# test_annotations.sh - tests whether a user is able to add pod annotations to a Fission environment deployment
|
||||
|
||||
TEST_ID=$(date +%s)
|
||||
ENV=python-${TEST_ID}
|
||||
FN=foo-${TEST_ID}
|
||||
RESOURCE_NS=default # Change to test-specific namespace once we support namespaced CRDs
|
||||
FUNCTION_NS=${FUNCTION_NAMESPACE:-fission-function}
|
||||
BUILDER_NS=fission-builder
|
||||
LIST_ANNOTATIONS=go-template='{{range $key,$value := .metadata.annotations}}{{$key}}: {{$value}}{{"\n"}}{{end}}'
|
||||
|
||||
# fs
|
||||
TEST_DIR=/tmp/${TEST_ID}
|
||||
ENV_SPEC_FILE=${TEST_DIR}/${ENV}.yaml
|
||||
FN_FILE=${TEST_DIR}/${FN}.yaml
|
||||
|
||||
log_exec() {
|
||||
cmd=$@
|
||||
echo "> ${cmd}"
|
||||
${cmd}
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
log "Cleaning up..."
|
||||
kubectl -n ${RESOURCE_NS} delete environment/${ENV} || true
|
||||
rm -rf ${TEST_DIR}
|
||||
|
||||
}
|
||||
|
||||
cleanup
|
||||
if [ -z "${TEST_NOCLEANUP:-}" ]; then
|
||||
trap cleanup EXIT
|
||||
else
|
||||
log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
|
||||
fi
|
||||
|
||||
if ! stat ${TEST_DIR} >/dev/null 2>&1 ; then
|
||||
mkdir ${TEST_DIR}
|
||||
fi
|
||||
|
||||
getPodName() {
|
||||
NS=$1
|
||||
POD=$2
|
||||
# find pod is ready to serve
|
||||
JSONPATH="{range .items[*]}{'\n'}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}"
|
||||
kubectl -n ${NS} get po -o jsonpath="$JSONPATH" \
|
||||
| grep "Ready=True" \
|
||||
| grep ${POD} \
|
||||
| head -n 1 \
|
||||
| cut -f1 -d":"
|
||||
}
|
||||
|
||||
# retry function adapted from:
|
||||
# https://unix.stackexchange.com/questions/82598/how-do-i-write-a-retry-logic-in-script-to-keep-retrying-to-run-it-upto-5-times/82610
|
||||
function retry {
|
||||
local n=1
|
||||
local max=5
|
||||
local delay=10 # pods take time to get ready
|
||||
while true; do
|
||||
"$@" && break || {
|
||||
if [[ ${n} -lt ${max} ]]; then
|
||||
((n++))
|
||||
echo "Command '$@' failed. Attempt $n/$max:"
|
||||
sleep ${delay};
|
||||
else
|
||||
>&2 echo "The command has failed after $n attempts."
|
||||
exit 1;
|
||||
fi
|
||||
}
|
||||
done
|
||||
}
|
||||
|
||||
# Deploy environment (using kubectl because the Fission cli does not support the container arguments)
|
||||
ANNOTATION_KEY="foo"
|
||||
ANNOTATION_VALUE="bar"
|
||||
echo "Writing environment config to $ENV_SPEC_FILE"
|
||||
cat > $ENV_SPEC_FILE <<- EOM
|
||||
apiVersion: fission.io/v1
|
||||
kind: Environment
|
||||
metadata:
|
||||
name: ${ENV}
|
||||
namespace: ${RESOURCE_NS}
|
||||
annotations:
|
||||
${ANNOTATION_KEY}: ${ANNOTATION_VALUE}
|
||||
spec:
|
||||
builder:
|
||||
command: build
|
||||
image: gcr.io/fission-ci/python-env-builder:test
|
||||
runtime:
|
||||
image: gcr.io/fission-ci/python-env:test
|
||||
version: 2
|
||||
poolsize: 1
|
||||
EOM
|
||||
log_exec kubectl -n ${RESOURCE_NS} apply -f ${ENV_SPEC_FILE}
|
||||
|
||||
sleep 15
|
||||
# Wait for runtime and build env to be deployed
|
||||
retry getPodName ${FUNCTION_NS} ${ENV} | grep '.\+'
|
||||
runtimePod=$(getPodName ${FUNCTION_NS} ${ENV})
|
||||
echo "function pod: ${runtimePod}."
|
||||
retry getPodName ${BUILDER_NS} ${ENV} | grep '.\+'
|
||||
buildPod=$(getPodName ${BUILDER_NS} ${ENV})
|
||||
echo "builder pod: ${buildPod}."
|
||||
|
||||
# Ensure pods are running/ready
|
||||
log "Waiting for ${FUNCTION_NS} ${ENV} to be available..."
|
||||
echo "> kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o \"${LIST_ANNOTATIONS}\""
|
||||
retry kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" > /dev/null
|
||||
log "Runtime pod ready."
|
||||
|
||||
log "Waiting for ${BUILDER_NS} ${ENV} to be available..."
|
||||
echo "> kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o \"${LIST_ANNOTATIONS}\""
|
||||
retry kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" > /dev/null
|
||||
log "Builder pod ready."
|
||||
|
||||
# Check if the annotation is set on the runtime pod
|
||||
status=0
|
||||
if kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" | grep "${ANNOTATION_KEY}: ${ANNOTATION_VALUE}"; then
|
||||
log "Runtime annotation is correct."
|
||||
else
|
||||
log "Runtime does not contain expected annotation: ${ANNOTATION_KEY}: ${ANNOTATION_VALUE}"
|
||||
echo "--- Runtime Env ---"
|
||||
kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" || true
|
||||
echo "--- End Runtime Env ---"
|
||||
status=5
|
||||
fi
|
||||
|
||||
# Check if the annotation is set on the builder pod
|
||||
if kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" | grep "${ANNOTATION_KEY}: ${ANNOTATION_VALUE}" ; then
|
||||
log "Builder annotation is correct."
|
||||
else
|
||||
log "Builder does not contain expected annotation: ${ANNOTATION_KEY}: ${ANNOTATION_VALUE}"
|
||||
echo "--- Builder Env ---"
|
||||
kubectl -n ${FUNCTION_NS} get pod ${runtimePod} -o "${LIST_ANNOTATIONS}" || true
|
||||
echo "--- End Builder Env ---"
|
||||
status=5
|
||||
fi
|
||||
exit ${status}
|
||||
Reference in New Issue
Block a user