diff --git a/test/benchmark/assets/envs/python_bjoern/README b/test/benchmark/assets/envs/python_bjoern/README new file mode 100644 index 00000000..1db3f9a5 --- /dev/null +++ b/test/benchmark/assets/envs/python_bjoern/README @@ -0,0 +1,42 @@ + +Fission Specs +============= + +This is a set of specifications for a Fission app. This includes functions, +environments, and triggers; we collectively call these things "resources". + +How to use these specs +---------------------- + +These specs are handled with the 'fission spec' command. See 'fission spec --help'. + +'fission spec apply' will "apply" all resources specified in this directory to your +cluster. That means it checks what resources exist on your cluster, what resources are +specified in the specs directory, and reconciles the difference by creating, updating or +deleting resources on the cluster. + +'fission spec apply' will also package up your source code (or compiled binaries) and +upload the archives to the cluster if needed. It uses 'ArchiveUploadSpec' resources in +this directory to figure out which files to archive. + +You can use 'fission spec apply --watch' to watch for file changes and continuously keep +the cluster updated. + +You can add YAMLs to this directory by writing them manually, but it's easier to generate +them. Use 'fission function create --spec' to generate a function spec, +'fission environment create --spec' to generate an environment spec, and so on. + +You can edit any of the files in this directory, except 'fission-deployment-config.yaml', +which contains a UID that you should never change. To apply your changes simply use +'fission spec apply'. + +fission-deployment-config.yaml +------------------------------ + +fission-deployment-config.yaml contains a UID. This UID is what fission uses to correlate +resources on the cluster to resources in this directory. + +All resources created by 'fission spec apply' are annotated with this UID. Resources on +the cluster that are _not_ annotated with this UID are never modified or deleted by +fission. + diff --git a/test/benchmark/assets/envs/python_bjoern/env.yaml b/test/benchmark/assets/envs/python_bjoern/env.yaml new file mode 100644 index 00000000..1a5c8889 --- /dev/null +++ b/test/benchmark/assets/envs/python_bjoern/env.yaml @@ -0,0 +1,20 @@ +apiVersion: fission.io/v1 +kind: Environment +metadata: + name: python + namespace: default +spec: + version: 2 + poolsize: 3 + TerminationGracePeriod: 5 + resources: + limits: + cpu: 300m + memory: 256Mi + requests: + cpu: 300m + memory: 256Mi + runtime: + image: fission/python-env + imagePullPolicy: IfNotPresent + diff --git a/test/benchmark/assets/envs/python_bjoern/fission-deployment-config.yaml b/test/benchmark/assets/envs/python_bjoern/fission-deployment-config.yaml new file mode 100644 index 00000000..6acad59d --- /dev/null +++ b/test/benchmark/assets/envs/python_bjoern/fission-deployment-config.yaml @@ -0,0 +1,7 @@ +# This file is generated by the 'fission spec init' command. +# See the README in this directory for background and usage information. +# Do not edit the UID below: that will break 'fission spec apply' +apiVersion: fission.io/v1 +kind: DeploymentConfig +name: envs +uid: a0f67a18-7d1d-4e50-9d9a-dab854e01106 diff --git a/test/benchmark/assets/envs/python_gevent/README b/test/benchmark/assets/envs/python_gevent/README new file mode 100644 index 00000000..1db3f9a5 --- /dev/null +++ b/test/benchmark/assets/envs/python_gevent/README @@ -0,0 +1,42 @@ + +Fission Specs +============= + +This is a set of specifications for a Fission app. This includes functions, +environments, and triggers; we collectively call these things "resources". + +How to use these specs +---------------------- + +These specs are handled with the 'fission spec' command. See 'fission spec --help'. + +'fission spec apply' will "apply" all resources specified in this directory to your +cluster. That means it checks what resources exist on your cluster, what resources are +specified in the specs directory, and reconciles the difference by creating, updating or +deleting resources on the cluster. + +'fission spec apply' will also package up your source code (or compiled binaries) and +upload the archives to the cluster if needed. It uses 'ArchiveUploadSpec' resources in +this directory to figure out which files to archive. + +You can use 'fission spec apply --watch' to watch for file changes and continuously keep +the cluster updated. + +You can add YAMLs to this directory by writing them manually, but it's easier to generate +them. Use 'fission function create --spec' to generate a function spec, +'fission environment create --spec' to generate an environment spec, and so on. + +You can edit any of the files in this directory, except 'fission-deployment-config.yaml', +which contains a UID that you should never change. To apply your changes simply use +'fission spec apply'. + +fission-deployment-config.yaml +------------------------------ + +fission-deployment-config.yaml contains a UID. This UID is what fission uses to correlate +resources on the cluster to resources in this directory. + +All resources created by 'fission spec apply' are annotated with this UID. Resources on +the cluster that are _not_ annotated with this UID are never modified or deleted by +fission. + diff --git a/test/benchmark/assets/envs/python_gevent/env.yaml b/test/benchmark/assets/envs/python_gevent/env.yaml new file mode 100644 index 00000000..17a99c41 --- /dev/null +++ b/test/benchmark/assets/envs/python_gevent/env.yaml @@ -0,0 +1,23 @@ +apiVersion: fission.io/v1 +kind: Environment +metadata: + name: python + namespace: default +spec: + version: 2 + poolsize: 3 + TerminationGracePeriod: 5 + resources: + limits: + cpu: 300m + memory: 256Mi + requests: + cpu: 300m + memory: 256Mi + runtime: + image: fission/python-env + imagePullPolicy: IfNotPresent + container: + env: + - name: WSGI_FRAMEWORK + value: "GEVENT" diff --git a/test/benchmark/assets/envs/python_gevent/fission-deployment-config.yaml b/test/benchmark/assets/envs/python_gevent/fission-deployment-config.yaml new file mode 100644 index 00000000..4be10c9c --- /dev/null +++ b/test/benchmark/assets/envs/python_gevent/fission-deployment-config.yaml @@ -0,0 +1,7 @@ +# This file is generated by the 'fission spec init' command. +# See the README in this directory for background and usage information. +# Do not edit the UID below: that will break 'fission spec apply' +apiVersion: fission.io/v1 +kind: DeploymentConfig +name: envs +uid: 1fc02a0c-7a51-4534-a86f-a8f397dfd1f2 diff --git a/test/benchmark/tests/concurrency-level/run.sh b/test/benchmark/tests/concurrency-level/run.sh index 53e19208..c3e93a4c 100755 --- a/test/benchmark/tests/concurrency-level/run.sh +++ b/test/benchmark/tests/concurrency-level/run.sh @@ -2,92 +2,95 @@ set -euo pipefail -ROOT=$(dirname $0)/../../../.. +ROOT=$(readlink -f ./../../../..) -for executorType in poolmgr newdeploy +for framework in python_bjoern python_gevent do - for concurrency in 100 250 500 750 1000 - do + for executorType in poolmgr newdeploy + do + for concurrency in 100 250 500 750 1000 + do - testDuration="60" - dirName="concurrency-${concurrency}-executor-${executorType}" + testDuration="60" + dirName="concurrency-${concurrency}-executor-${executorType}" - # remove old data - rm -rf ${dirName} - mkdir ${dirName} - pushd ${dirName} + # remove old data + rm -rf ${dirName} + mkdir ${dirName} + pushd ${dirName} - # run multiple iterations to reduce impact of imbalance of pod distribution. - for iteration in {1..10} - do + # run multiple iterations to reduce impact of imbalance of pod distribution. + for iteration in {1..10} + do - # Create a hello world function in nodejs, test it with an http trigger - echo "Pre-test cleanup" - fission env delete --name python || true + # Create a hello world function in nodejs, test it with an http trigger + echo "Pre-test cleanup" + fission env delete --name python || true - echo "Creating python env" - # Use short grace period time to speed up resource recycle time - # Use high min/max CPU so that K8S will distribute pod in different nodes - fission env create --name python --version 2 --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256 - trap "fission env delete --name python" EXIT + echo "Creating python env" + # Use short grace period time to speed up resource recycle time + # Use high min/max CPU so that K8S will distribute pod in different nodes + fission spec apply --specdir $ROOT/test/benchmark/assets/envs/$framework + trap "fission env delete --name python" EXIT - sleep 30 + sleep 30 - fn=python-hello-$(date +%s) + fn=python-hello-$(date +%s) - echo "Creating package" - rm -rf pkg.zip pkg/ || true - mkdir pkg - cp ../../../assets/hello.py pkg/hello.py + echo "Creating package" + rm -rf pkg.zip pkg/ || true + mkdir pkg + cp $ROOT/test/benchmark/assets/hello.py pkg/hello.py - zip -jr pkg.zip pkg/ - pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2) + zip -jr pkg.zip pkg/ + pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2) - echo "Creating function" - fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3 + echo "Creating function" + fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3 - echo "Creating route" - fission route create --function $fn --url /$fn --method GET + echo "Creating route" + fission route create --function $fn --url /$fn --method GET - echo "Waiting for router to catch up" - sleep 5 + echo "Waiting for router to catch up" + sleep 5 - fnEndpoint="http://$FISSION_ROUTER/$fn" - js="sample.js" - rawFile="raw-${iteration}.json" - rawUsageReport="raw-usage.txt" + fnEndpoint="http://$FISSION_ROUTER/$fn" + js="sample.js" + rawFile="raw-${iteration}.json" + rawUsageReport="raw-usage.txt" - k6 run \ - -e FN_ENDPOINT="${fnEndpoint}" \ - --duration "${testDuration}s" \ - --rps ${concurrency} \ - --vus ${concurrency} \ - --no-connection-reuse \ - --out json="${rawFile}" \ - --summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \ - ../${js} >> ${rawUsageReport} + k6 run \ + -e FN_ENDPOINT="${fnEndpoint}" \ + --duration "${testDuration}s" \ + --rps ${concurrency} \ + --vus ${concurrency} \ + --no-connection-reuse \ + --out json="${rawFile}" \ + --summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30)" \ + ../${js} >> ${rawUsageReport} + + echo "Clean up" + fission fn delete --name ${fn} + fission env delete --name python + fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name + fission pkg delete --name ${pkgName} + rm -rf pkg.zip pkg - echo "Clean up" - fission fn delete --name ${fn} - fission env delete --name python - fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name - fission pkg delete --name ${pkgName} - rm -rf pkg.zip pkg + kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true + kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true - kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true - kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true + echo "All done." + done - echo "All done." - done + usageReport="usage.txt" + cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport} - usageReport="usage.txt" - cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport} + popd - popd + # generate report after iterations are over + outImage="${dirName}.png" + picasso -file ${dirName} -format png -o ${outImage} - # generate report after iterations are over - outImage="${dirName}.png" - picasso -file ${dirName} -format png -o ${outImage} - - done + done + done done