Ta-Ching ChenandGitHub
51b264e8ca
The pool manager keeps terminating function pod periodically even there are traffic to the function. The root cause is that executor, poolmgr, newdeploy manage their own functionServiceCache separately. And when router taps a function, executor updates the access time of the function service entry in its own cache without notifying executor types to do the update as well. Hence, the access time of function service entry in poolmanager cache never gets updated. Due to the access time never gets updated, the idle pod reaper in poolmanager then thinks the function pod is in idle state and recycle it. This PR removes the cache in executor itself, and when router tries to tap a function, executor will call executor type to tap the function and update access time.
90 lines
3.1 KiB
Bash
Executable File
90 lines
3.1 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -euo pipefail
|
|
source $(dirname $0)/../../utils.sh
|
|
|
|
TEST_ID=$(generate_test_id)
|
|
echo "TEST_ID = $TEST_ID"
|
|
|
|
env=python-$TEST_ID
|
|
fn=hellopython-$TEST_ID
|
|
ROOT=$(dirname $0)/../../..
|
|
|
|
cleanup() {
|
|
log "Cleaning up..."
|
|
clean_resource_by_id $TEST_ID
|
|
}
|
|
|
|
if [ -z "${TEST_NOCLEANUP:-}" ]; then
|
|
trap cleanup EXIT
|
|
else
|
|
log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
|
|
fi
|
|
|
|
log "Creating Python env $env"
|
|
fission env create --name $env --image $PYTHON_RUNTIME_IMAGE --period 5
|
|
|
|
log "Creating function ${fn}-nd, ${fn}-gpm"
|
|
fission fn create --name ${fn}-nd --env $env --code $ROOT/examples/python/hello.py --minscale 0 --maxscale 2 --executortype newdeploy
|
|
fission fn create --name ${fn}-gpm --env $env --code $ROOT/examples/python/hello.py
|
|
|
|
log "Creating route for function $fn"
|
|
fission route create --function ${fn}-nd --url /${fn}-nd --method GET
|
|
fission route create --function ${fn}-gpm --url /${fn}-gpm --method GET
|
|
|
|
log "Waiting for update to catch up"
|
|
sleep 5
|
|
|
|
timeout 60 bash -c "test_fn ${fn}-nd 'world'"
|
|
timeout 60 bash -c "test_fn ${fn}-gpm 'world'"
|
|
|
|
log "Waiting for idle pod reaper to recycle resources"
|
|
# the LIST_OLD function list fsvc older than 2 mins
|
|
# so in worst case, we need to wait for up to 4 mins + some buffer
|
|
sleep 300
|
|
|
|
# The replicas of function deployment should be 0 due to minScale = 0
|
|
ndDeployReplicas=$(kubectl -n $FUNCTION_NAMESPACE get deploy -l functionName=${fn}-nd -ojsonpath='{.items[0].spec.replicas}')
|
|
if [ "$ndDeployReplicas" -ne "0" ]
|
|
then
|
|
log "Failed to reap idle function pod for function ${fn}-nd. replicas should be 0 but got $ndDeployReplicas"
|
|
kubectl -n $FUNCTION_NAMESPACE get deploy -l functionName=${fn}-nd -o yaml
|
|
exit 1
|
|
fi
|
|
|
|
set +o pipefail
|
|
gpmNumberOfPod=$(kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm | grep Running | wc -l)
|
|
set -o pipefail
|
|
if [ "$gpmNumberOfPod" -ne "0" ]
|
|
then
|
|
log "Failed to reap idle function pod for function ${fn}-gpm. replicas should be 0 but got $gpmNumberOfPod"
|
|
kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm -o yaml
|
|
exit 1
|
|
fi
|
|
|
|
# The executor will scale the deployment from 0 to minScale.
|
|
# If minScale is 0 then scale to 1 instead.
|
|
timeout 60 bash -c "test_fn ${fn}-nd 'world'"
|
|
timeout 60 bash -c "test_fn ${fn}-gpm 'world'"
|
|
|
|
# The replicas of function deployment should be scaled to 1 due to minScale is 0
|
|
ndDeployReplicas=$(kubectl -n $FUNCTION_NAMESPACE get deploy -l functionName=${fn}-nd -ojsonpath='{.items[0].spec.replicas}')
|
|
if [ "$ndDeployReplicas" -ne "1" ]
|
|
then
|
|
log "Failed to scale function pod for function ${fn}-nd. replicas should be 1 but got $ndDeployReplicas"
|
|
kubectl -n $FUNCTION_NAMESPACE get deploy -l functionName=${fn}-nd -o yaml
|
|
exit 1
|
|
fi
|
|
|
|
set +o pipefail
|
|
gpmNumberOfPod=$(kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm | grep Running | wc -l)
|
|
set -o pipefail
|
|
if [ "$gpmNumberOfPod" -ne "1" ]
|
|
then
|
|
log "Failed to scale function pod for function ${fn}-gpm. replicas should be 1 but got $gpmNumberOfPod"
|
|
kubectl -n $FUNCTION_NAMESPACE get pod -l functionName=${fn}-gpm -o yaml
|
|
exit 1
|
|
fi
|
|
|
|
log "Test PASSED"
|