diff --git a/hack/runtests.sh b/hack/runtests.sh index 6c26ca7f..d61e63f3 100755 --- a/hack/runtests.sh +++ b/hack/runtests.sh @@ -18,8 +18,8 @@ else fi fi -go test -v -i $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go') +go test -v -i $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v 'benchmark') # The executor unit test only works with NodePort-type services for # now. So disable it for our travis ci tests. -go test -v $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v executor) +go test -v $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v executor | grep -v 'benchmark') diff --git a/test/benchmark/Dockerfile b/test/benchmark/Dockerfile new file mode 100644 index 00000000..c2de0d8c --- /dev/null +++ b/test/benchmark/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.10.1 AS go-builder +WORKDIR /go +RUN go get github.com/wcharczuk/go-chart +COPY picasso.go /go +RUN CGO_ENABLE=0 GOOS=linux GOARCH=amd64 go build -o picasso . + +FROM loadimpact/k6 +WORKDIR /fission-bench +COPY --from=go-builder /go/picasso /usr/local/bin/picasso +RUN apk --update add --no-cache bash curl +RUN curl -Lo fission https://github.com/fission/fission/releases/download/$(curl --silent "https://api.github.com/repos/fission/fission/releases/latest" | grep "tag_name" |sed -E 's/.*"([^"]+)".*/\1/')/fission-cli-linux && chmod +x fission && mv fission /usr/local/bin/ + +ENTRYPOINT ["sh"] diff --git a/test/benchmark/assets/hello.py b/test/benchmark/assets/hello.py new file mode 100644 index 00000000..2c3a3088 --- /dev/null +++ b/test/benchmark/assets/hello.py @@ -0,0 +1,2 @@ +def main(): + return "Hello, world!\n" diff --git a/test/benchmark/picasso.go b/test/benchmark/picasso.go new file mode 100644 index 00000000..90948981 --- /dev/null +++ b/test/benchmark/picasso.go @@ -0,0 +1,249 @@ +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/wcharczuk/go-chart" +) + +var ( + title *string + file *string + outputFile *string + outputFormat *string +) + +const ( + PNG = "png" + SVG = "svg" +) + +type ( + Tags struct { + Group string `json:"group"` + Iter string `json:"iter"` + Method string `json:"method"` + Name string `json:"name"` + Proto string `json:"proto"` + Status string `json:"status"` + URL string `json:"url"` + Vu string `json:"vu"` + } + + Data struct { + Time time.Time `json:"time"` + Value float64 `json:"value"` + Tags Tags `json:"tags"` + } + + MetricPoint struct { + Type string `json:"type"` + Data Data `json:"data"` + Metric string `json:"metric"` + } +) + +func init() { + title = flag.String("title", "Fission Benchmark", "Chart title") + file = flag.String("file", "", "Metric json file") + outputFile = flag.String("o", "chart.png", "Output file name") + outputFormat = flag.String("format", PNG, "Format of output file (png or svg)") + flag.Parse() +} + +func generateContinuousSeries(file string) chart.Series { + f, err := os.OpenFile(file, os.O_RDONLY, 0644) + if err != nil { + fmt.Printf("Failed to open file metric point: %v", err) + return nil + } + defer f.Close() + + reader := bufio.NewReader(f) + + var points []*MetricPoint + var xVals []float64 + var yVals []float64 + + var initTime *time.Time + + for { + l, _, err := reader.ReadLine() + if err == io.EOF { + break + } + + point := &MetricPoint{} + err = json.Unmarshal(l, point) + if err != nil { + fmt.Printf("Failed to parse metric point: %v -> %v", err, string(l)) + return nil + } + + if point.Type != "Point" || point.Metric != "http_req_duration" { + continue + } + + points = append(points, point) + + if initTime == nil { + initTime = &point.Data.Time + } + + timeSinceStart := point.Data.Time.Sub(*initTime).Seconds() + xVals = append(xVals, timeSinceStart) + yVals = append(yVals, point.Data.Value) + } + + return chart.ContinuousSeries{ + Style: chart.Style{ + Show: true, + StrokeColor: chart.GetDefaultColor(0).WithAlpha(64), + FillColor: chart.GetDefaultColor(0).WithAlpha(64), + }, + XValues: xVals, + YValues: yVals, + } +} + +func generateChart(title string, file string, format chart.RendererProvider, series []chart.Series) error { + if series == nil { + return errors.New("Series cannot be nil") + } + + cs := chart.ConcatSeries(series) + + graph := chart.Chart{ + Title: title, + TitleStyle: chart.StyleShow(), + Background: chart.Style{ + Padding: chart.Box{ + Top: 50, + Left: 25, + Right: 25, + Bottom: 10, + }, + }, + XAxis: chart.XAxis{ + Name: "Time (s)", + NameStyle: chart.StyleShow(), + Style: chart.StyleShow(), + Range: &chart.ContinuousRange{ + Min: 0, + }, + ValueFormatter: func(v interface{}) string { + return fmt.Sprintf("%.2f s", v.(float64)) + }, + }, + YAxis: chart.YAxis{ + Name: "Response Time (ms)", + NameStyle: chart.StyleShow(), + Style: chart.StyleShow(), + Range: &chart.ContinuousRange{ + Min: 0, + }, + ValueFormatter: func(v interface{}) string { + return fmt.Sprintf("%d ms", int(v.(float64))) + }, + }, + Series: cs, + } + + buffer := bytes.NewBuffer([]byte{}) + err := graph.Render(format, buffer) + if err != nil { + return err + } + + err = ioutil.WriteFile(file, buffer.Bytes(), 0644) + if err != nil { + return err + } + + return nil +} + +func listJsonFiles(path string) ([]string, error) { + fi, err := os.Stat(*file) + if err != nil { + return nil, err + } + + if fi.Mode().IsRegular() { + return []string{path}, nil + } + + var files []string + + err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if strings.HasSuffix(path, ".json") && !info.IsDir() { + files = append(files, path) + return nil + } + + return nil + }) + + return files, nil +} + +func main() { + + if file == nil || len(*file) == 0 { + fmt.Println("Please provide metric json file name") + return + } + + var format chart.RendererProvider + + if outputFormat == nil { + format = chart.PNG + } else { + switch strings.ToLower(*outputFormat) { + case PNG: + format = chart.PNG + case SVG: + format = chart.SVG + default: + fmt.Println("Unknown format, use png as output format") + *outputFormat = PNG + } + } + + if len(*outputFile) == 0 { + fmt.Println("Please output chart png file name") + return + } + + var series []chart.Series + + files, err := listJsonFiles(*file) + if err != nil { + fmt.Printf("Failed to get file information: %v", err) + return + } + + for _, f := range files { + series = append(series, generateContinuousSeries(f)) + } + + err = generateChart(*title, *outputFile, format, series) + if err != nil { + fmt.Printf("Failed to generate chart: %v\n", err) + } +} diff --git a/test/benchmark/tests/burst-load/run.sh b/test/benchmark/tests/burst-load/run.sh new file mode 100755 index 00000000..c3254d58 --- /dev/null +++ b/test/benchmark/tests/burst-load/run.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +set -euo pipefail + +ROOT=$(dirname $0)/../../../.. + +for executorType in poolmgr newdeploy +do + dirName="burst-load-executor-${executorType}" + + # remove old data + rm -rf ${dirName} + mkdir ${dirName} + pushd ${dirName} + + # run multiple iterations to reduce impact of imbalance of pod distribution. + for iteration in {1..10} + do + + # Create a hello world function in nodejs, test it with an http trigger + echo "Pre-test cleanup" + fission env delete --name python || true + + echo "Creating python env" + # Use short grace period time to speed up resource recycle time + # Use high min/max CPU so that K8S will distribute pod in different nodes + fission env create --name python --version 2 --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256 + trap "fission env delete --name python" EXIT + + sleep 30 + + fn=python-hello-$(date +%s) + + echo "Creating package" + rm -rf pkg.zip pkg/ || true + mkdir pkg + cp ../../../assets/hello.py pkg/hello.py + + zip -jr pkg.zip pkg/ + pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2) + + echo "Creating function" + fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3 + + echo "Creating route" + fission route create --function $fn --url /$fn --method GET + + echo "Waiting for router to catch up" + sleep 5 + + fnEndpoint="http://$FISSION_ROUTER/$fn" + js="sample.js" + rawFile="raw-${iteration}.json" + rawUsageReport="raw-usage.txt" + + # Stage 1: 15s, 10 vus + # Stage 2: 45s, 500 vus + # Please check sample.js for more detail. + k6 run \ + -e FN_ENDPOINT="${fnEndpoint}" \ + --no-connection-reuse \ + --out json="${rawFile}" \ + --summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \ + ../${js} >> ${rawUsageReport} + + echo "Clean up" + fission fn delete --name ${fn} + fission env delete --name python + fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name + fission pkg delete --name ${pkgName} + rm -rf pkg.zip pkg + + kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true + kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true + + echo "All done." + done + + usageReport="usage.txt" + cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport} + + popd + + # generate report after iterations are over + outImage="${dirName}.png" + picasso -file ${dirName} -format png -o ${outImage} + +done diff --git a/test/benchmark/tests/burst-load/sample.js b/test/benchmark/tests/burst-load/sample.js new file mode 100644 index 00000000..40876fe6 --- /dev/null +++ b/test/benchmark/tests/burst-load/sample.js @@ -0,0 +1,17 @@ +import http from "k6/http"; +import { check } from "k6"; + +export let options = { + stages: [ + { duration: "15s", target: 10 }, + { duration: "45s", target: 500 }, + ] +}; + +export default function() { + let params = { timeout: 30 } + let res = http.get(`${__ENV.FN_ENDPOINT}`) + check(res, { + "status is 200": (r) => r.status === 200 + }); +}; diff --git a/test/benchmark/tests/concurrency-level/run.sh b/test/benchmark/tests/concurrency-level/run.sh new file mode 100755 index 00000000..53e19208 --- /dev/null +++ b/test/benchmark/tests/concurrency-level/run.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -euo pipefail + +ROOT=$(dirname $0)/../../../.. + +for executorType in poolmgr newdeploy +do + for concurrency in 100 250 500 750 1000 + do + + testDuration="60" + dirName="concurrency-${concurrency}-executor-${executorType}" + + # remove old data + rm -rf ${dirName} + mkdir ${dirName} + pushd ${dirName} + + # run multiple iterations to reduce impact of imbalance of pod distribution. + for iteration in {1..10} + do + + # Create a hello world function in nodejs, test it with an http trigger + echo "Pre-test cleanup" + fission env delete --name python || true + + echo "Creating python env" + # Use short grace period time to speed up resource recycle time + # Use high min/max CPU so that K8S will distribute pod in different nodes + fission env create --name python --version 2 --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256 + trap "fission env delete --name python" EXIT + + sleep 30 + + fn=python-hello-$(date +%s) + + echo "Creating package" + rm -rf pkg.zip pkg/ || true + mkdir pkg + cp ../../../assets/hello.py pkg/hello.py + + zip -jr pkg.zip pkg/ + pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2) + + echo "Creating function" + fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3 + + echo "Creating route" + fission route create --function $fn --url /$fn --method GET + + echo "Waiting for router to catch up" + sleep 5 + + fnEndpoint="http://$FISSION_ROUTER/$fn" + js="sample.js" + rawFile="raw-${iteration}.json" + rawUsageReport="raw-usage.txt" + + k6 run \ + -e FN_ENDPOINT="${fnEndpoint}" \ + --duration "${testDuration}s" \ + --rps ${concurrency} \ + --vus ${concurrency} \ + --no-connection-reuse \ + --out json="${rawFile}" \ + --summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \ + ../${js} >> ${rawUsageReport} + + echo "Clean up" + fission fn delete --name ${fn} + fission env delete --name python + fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name + fission pkg delete --name ${pkgName} + rm -rf pkg.zip pkg + + kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true + kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true + + echo "All done." + done + + usageReport="usage.txt" + cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport} + + popd + + # generate report after iterations are over + outImage="${dirName}.png" + picasso -file ${dirName} -format png -o ${outImage} + + done +done diff --git a/test/benchmark/tests/concurrency-level/sample.js b/test/benchmark/tests/concurrency-level/sample.js new file mode 100644 index 00000000..a26dac42 --- /dev/null +++ b/test/benchmark/tests/concurrency-level/sample.js @@ -0,0 +1,10 @@ +import http from "k6/http"; +import { check } from "k6"; + +export default function() { + let params = { timeout: 30 } + let res = http.get(`${__ENV.FN_ENDPOINT}`) + check(res, { + "status is 200": (r) => r.status === 200 + }); +}; diff --git a/test/benchmark/tests/package-size/run.sh b/test/benchmark/tests/package-size/run.sh new file mode 100755 index 00000000..d7700a45 --- /dev/null +++ b/test/benchmark/tests/package-size/run.sh @@ -0,0 +1,117 @@ +#!/bin/bash + +set -euo pipefail + +ROOT=$(dirname $0)/../../../.. + +for executorType in poolmgr newdeploy +do + for packagesize in 0 1 5 10 15 20 + do + + testDuration="5" + dirName="package-size-${packagesize}-executor-${executorType}" + + # remove old data + rm -rf ${dirName} + mkdir ${dirName} + pushd ${dirName} + + # run multiple iterations to reduce impact of imbalance of pod distribution. + for iteration in {1..10} + do + + # Create a hello world function in nodejs, test it with an http trigger + echo "Pre-test cleanup" + fission env delete --name python || true + + echo "Creating python env" + # Use short grace period time to speed up resource recycle time + # Use high min/max CPU so that K8S will distribute pod in different nodes + + version=2 + + if [[ "${packagesize}" == "0" ]] + then + version=1 + fi + + fission env create --name python --version ${version} --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256 + + trap "fission env delete --name python" EXIT + + sleep 30 + + fn=python-hello-$(date +%s) + + pkgName="" + + if [[ "${packagesize}" == "0" ]] + then + echo "Creating function" + fission fn create --name $fn --env python --code ../../../assets/hello.py --executortype ${executorType} --minscale 3 --maxscale 3 + else + echo "Creating package" + rm -rf pkg.zip pkg/ || true + mkdir pkg + cp ../../../assets/hello.py pkg/hello.py + + # Create empty file with give size to simulate different size of package + truncate -s ${packagesize}MiB pkg/foo + + zip -jr pkg.zip pkg/ + pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2) + + echo "Creating function" + fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3 + fi + + echo "Creating route" + fission route create --function $fn --url /$fn --method GET + + echo "Waiting for router to catch up" + sleep 5 + + fnEndpoint="http://$FISSION_ROUTER/$fn" + js="sample.js" + rawFile="raw-${iteration}.json" + rawUsageReport="raw-usage.txt" + + k6 run \ + -e FN_ENDPOINT="${fnEndpoint}" \ + --duration "${testDuration}s" \ + --rps 1 \ + --vus 1 \ + --no-connection-reuse \ + --out json="${rawFile}" \ + --summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \ + ../${js} >> ${rawUsageReport} + + echo "Clean up" + fission fn delete --name ${fn} + fission env delete --name python + fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name + + if [[ ! -z "${pkgName}" ]] + then + fission pkg delete --name ${pkgName} || true + rm -rf pkg.zip pkg + fi + + kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true + kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true + + echo "All done." + done + + usageReport="usage.txt" + cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport} + + popd + + # generate report after iterations are over + outImage="${dirName}.png" + picasso -file ${dirName} -format png -o ${outImage} + + done +done diff --git a/test/benchmark/tests/package-size/sample.js b/test/benchmark/tests/package-size/sample.js new file mode 100644 index 00000000..a26dac42 --- /dev/null +++ b/test/benchmark/tests/package-size/sample.js @@ -0,0 +1,10 @@ +import http from "k6/http"; +import { check } from "k6"; + +export default function() { + let params = { timeout: 30 } + let res = http.get(`${__ENV.FN_ENDPOINT}`) + check(res, { + "status is 200": (r) => r.status === 200 + }); +};