Add benchmark script (#666)

hack/runtests.sh

@@ -18,8 +18,8 @@ else
     fi
 fi
 
-go test -v -i $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go')
+go test -v -i $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v 'benchmark')
 
 # The executor unit test only works with NodePort-type services for
 # now. So disable it for our travis ci tests.
-go test -v $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v executor)
+go test -v $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v executor | grep -v 'benchmark')

test/benchmark/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.10.1 AS go-builder
+WORKDIR /go
+RUN go get github.com/wcharczuk/go-chart
+COPY picasso.go /go
+RUN CGO_ENABLE=0 GOOS=linux GOARCH=amd64 go build -o picasso .
+
+FROM loadimpact/k6
+WORKDIR /fission-bench
+COPY --from=go-builder /go/picasso /usr/local/bin/picasso
+RUN apk --update add --no-cache bash curl
+RUN curl -Lo fission https://github.com/fission/fission/releases/download/$(curl --silent "https://api.github.com/repos/fission/fission/releases/latest" | grep "tag_name" |sed -E 's/.*"([^"]+)".*/\1/')/fission-cli-linux && chmod +x fission && mv fission /usr/local/bin/
+
+ENTRYPOINT ["sh"]

test/benchmark/assets/hello.py

@@ -0,0 +1,2 @@
+def main():
+    return "Hello, world!\n"

test/benchmark/picasso.go

@@ -0,0 +1,249 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/wcharczuk/go-chart"
+)
+
+var (
+	title        *string
+	file         *string
+	outputFile   *string
+	outputFormat *string
+)
+
+const (
+	PNG = "png"
+	SVG = "svg"
+)
+
+type (
+	Tags struct {
+		Group  string `json:"group"`
+		Iter   string `json:"iter"`
+		Method string `json:"method"`
+		Name   string `json:"name"`
+		Proto  string `json:"proto"`
+		Status string `json:"status"`
+		URL    string `json:"url"`
+		Vu     string `json:"vu"`
+	}
+
+	Data struct {
+		Time  time.Time `json:"time"`
+		Value float64   `json:"value"`
+		Tags  Tags      `json:"tags"`
+	}
+
+	MetricPoint struct {
+		Type   string `json:"type"`
+		Data   Data   `json:"data"`
+		Metric string `json:"metric"`
+	}
+)
+
+func init() {
+	title = flag.String("title", "Fission Benchmark", "Chart title")
+	file = flag.String("file", "", "Metric json file")
+	outputFile = flag.String("o", "chart.png", "Output file name")
+	outputFormat = flag.String("format", PNG, "Format of output file (png or svg)")
+	flag.Parse()
+}
+
+func generateContinuousSeries(file string) chart.Series {
+	f, err := os.OpenFile(file, os.O_RDONLY, 0644)
+	if err != nil {
+		fmt.Printf("Failed to open file metric point: %v", err)
+		return nil
+	}
+	defer f.Close()
+
+	reader := bufio.NewReader(f)
+
+	var points []*MetricPoint
+	var xVals []float64
+	var yVals []float64
+
+	var initTime *time.Time
+
+	for {
+		l, _, err := reader.ReadLine()
+		if err == io.EOF {
+			break
+		}
+
+		point := &MetricPoint{}
+		err = json.Unmarshal(l, point)
+		if err != nil {
+			fmt.Printf("Failed to parse metric point: %v -> %v", err, string(l))
+			return nil
+		}
+
+		if point.Type != "Point" || point.Metric != "http_req_duration" {
+			continue
+		}
+
+		points = append(points, point)
+
+		if initTime == nil {
+			initTime = &point.Data.Time
+		}
+
+		timeSinceStart := point.Data.Time.Sub(*initTime).Seconds()
+		xVals = append(xVals, timeSinceStart)
+		yVals = append(yVals, point.Data.Value)
+	}
+
+	return chart.ContinuousSeries{
+		Style: chart.Style{
+			Show:        true,
+			StrokeColor: chart.GetDefaultColor(0).WithAlpha(64),
+			FillColor:   chart.GetDefaultColor(0).WithAlpha(64),
+		},
+		XValues: xVals,
+		YValues: yVals,
+	}
+}
+
+func generateChart(title string, file string, format chart.RendererProvider, series []chart.Series) error {
+	if series == nil {
+		return errors.New("Series cannot be nil")
+	}
+
+	cs := chart.ConcatSeries(series)
+
+	graph := chart.Chart{
+		Title:      title,
+		TitleStyle: chart.StyleShow(),
+		Background: chart.Style{
+			Padding: chart.Box{
+				Top:    50,
+				Left:   25,
+				Right:  25,
+				Bottom: 10,
+			},
+		},
+		XAxis: chart.XAxis{
+			Name:      "Time (s)",
+			NameStyle: chart.StyleShow(),
+			Style:     chart.StyleShow(),
+			Range: &chart.ContinuousRange{
+				Min: 0,
+			},
+			ValueFormatter: func(v interface{}) string {
+				return fmt.Sprintf("%.2f s", v.(float64))
+			},
+		},
+		YAxis: chart.YAxis{
+			Name:      "Response Time (ms)",
+			NameStyle: chart.StyleShow(),
+			Style:     chart.StyleShow(),
+			Range: &chart.ContinuousRange{
+				Min: 0,
+			},
+			ValueFormatter: func(v interface{}) string {
+				return fmt.Sprintf("%d ms", int(v.(float64)))
+			},
+		},
+		Series: cs,
+	}
+
+	buffer := bytes.NewBuffer([]byte{})
+	err := graph.Render(format, buffer)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(file, buffer.Bytes(), 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func listJsonFiles(path string) ([]string, error) {
+	fi, err := os.Stat(*file)
+	if err != nil {
+		return nil, err
+	}
+
+	if fi.Mode().IsRegular() {
+		return []string{path}, nil
+	}
+
+	var files []string
+
+	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if strings.HasSuffix(path, ".json") && !info.IsDir() {
+			files = append(files, path)
+			return nil
+		}
+
+		return nil
+	})
+
+	return files, nil
+}
+
+func main() {
+
+	if file == nil || len(*file) == 0 {
+		fmt.Println("Please provide metric json file name")
+		return
+	}
+
+	var format chart.RendererProvider
+
+	if outputFormat == nil {
+		format = chart.PNG
+	} else {
+		switch strings.ToLower(*outputFormat) {
+		case PNG:
+			format = chart.PNG
+		case SVG:
+			format = chart.SVG
+		default:
+			fmt.Println("Unknown format, use png as output format")
+			*outputFormat = PNG
+		}
+	}
+
+	if len(*outputFile) == 0 {
+		fmt.Println("Please output chart png file name")
+		return
+	}
+
+	var series []chart.Series
+
+	files, err := listJsonFiles(*file)
+	if err != nil {
+		fmt.Printf("Failed to get file information: %v", err)
+		return
+	}
+
+	for _, f := range files {
+		series = append(series, generateContinuousSeries(f))
+	}
+
+	err = generateChart(*title, *outputFile, format, series)
+	if err != nil {
+		fmt.Printf("Failed to generate chart: %v\n", err)
+	}
+}

test/benchmark/tests/burst-load/run.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ROOT=$(dirname $0)/../../../..
+
+for executorType in poolmgr newdeploy
+do
+    dirName="burst-load-executor-${executorType}"
+
+    # remove old data
+    rm -rf ${dirName}
+    mkdir ${dirName}
+    pushd ${dirName}
+
+    # run multiple iterations to reduce impact of imbalance of pod distribution.
+    for iteration in {1..10}
+    do
+
+        # Create a hello world function in nodejs, test it with an http trigger
+        echo "Pre-test cleanup"
+        fission env delete --name python || true
+
+        echo "Creating python env"
+        # Use short grace period time to speed up resource recycle time
+        # Use high min/max CPU so that K8S will distribute pod in different nodes
+        fission env create --name python --version 2 --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256
+        trap "fission env delete --name python" EXIT
+
+        sleep 30
+
+        fn=python-hello-$(date +%s)
+
+        echo "Creating package"
+        rm -rf pkg.zip pkg/ || true
+        mkdir pkg
+        cp ../../../assets/hello.py pkg/hello.py
+
+        zip -jr pkg.zip pkg/
+        pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2)
+
+        echo "Creating function"
+        fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3
+
+        echo "Creating route"
+        fission route create --function $fn --url /$fn --method GET
+
+        echo "Waiting for router to catch up"
+        sleep 5
+
+        fnEndpoint="http://$FISSION_ROUTER/$fn"
+        js="sample.js"
+        rawFile="raw-${iteration}.json"
+        rawUsageReport="raw-usage.txt"
+
+        # Stage 1: 15s, 10 vus
+        # Stage 2: 45s, 500 vus
+        # Please check sample.js for more detail.
+        k6 run \
+            -e FN_ENDPOINT="${fnEndpoint}" \
+            --no-connection-reuse \
+            --out json="${rawFile}" \
+            --summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \
+            ../${js} >> ${rawUsageReport}
+
+        echo "Clean up"
+        fission fn delete --name ${fn}
+        fission env delete --name python
+        fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name
+        fission pkg delete --name ${pkgName}
+        rm -rf pkg.zip pkg
+
+        kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
+        kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
+
+        echo "All done."
+    done
+
+    usageReport="usage.txt"
+    cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport}
+
+    popd
+
+    # generate report after iterations are over
+    outImage="${dirName}.png"
+    picasso -file ${dirName} -format png -o ${outImage}
+
+done

test/benchmark/tests/burst-load/sample.js

@@ -0,0 +1,17 @@
+import http from "k6/http";
+import { check } from "k6";
+
+export let options = {
+    stages: [
+        { duration: "15s", target: 10 },
+        { duration: "45s", target: 500  },
+    ]
+};
+
+export default function() {
+    let params = { timeout: 30 }
+    let res = http.get(`${__ENV.FN_ENDPOINT}`)
+    check(res, {
+        "status is 200": (r) => r.status === 200
+    });
+};

test/benchmark/tests/concurrency-level/run.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ROOT=$(dirname $0)/../../../..
+
+for executorType in poolmgr newdeploy
+do
+    for concurrency in 100 250 500 750 1000
+    do
+
+        testDuration="60"
+        dirName="concurrency-${concurrency}-executor-${executorType}"
+
+        # remove old data
+        rm -rf ${dirName}
+        mkdir ${dirName}
+        pushd ${dirName}
+
+        # run multiple iterations to reduce impact of imbalance of pod distribution.
+        for iteration in {1..10}
+        do
+
+            # Create a hello world function in nodejs, test it with an http trigger
+            echo "Pre-test cleanup"
+            fission env delete --name python || true
+
+            echo "Creating python env"
+            # Use short grace period time to speed up resource recycle time
+            # Use high min/max CPU so that K8S will distribute pod in different nodes
+            fission env create --name python --version 2 --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256
+            trap "fission env delete --name python" EXIT
+
+            sleep 30
+
+            fn=python-hello-$(date +%s)
+
+            echo "Creating package"
+            rm -rf pkg.zip pkg/ || true
+            mkdir pkg
+            cp ../../../assets/hello.py pkg/hello.py
+
+            zip -jr pkg.zip pkg/
+            pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2)
+
+            echo "Creating function"
+            fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3
+
+            echo "Creating route"
+            fission route create --function $fn --url /$fn --method GET
+
+            echo "Waiting for router to catch up"
+            sleep 5
+
+            fnEndpoint="http://$FISSION_ROUTER/$fn"
+            js="sample.js"
+            rawFile="raw-${iteration}.json"
+            rawUsageReport="raw-usage.txt"
+
+            k6 run \
+                -e FN_ENDPOINT="${fnEndpoint}" \
+                --duration "${testDuration}s" \
+                --rps ${concurrency} \
+                --vus ${concurrency} \
+                --no-connection-reuse \
+                --out json="${rawFile}" \
+                --summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \
+                ../${js} >> ${rawUsageReport}
+
+            echo "Clean up"
+            fission fn delete --name ${fn}
+            fission env delete --name python
+            fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name
+            fission pkg delete --name ${pkgName}
+            rm -rf pkg.zip pkg
+
+            kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
+            kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
+
+            echo "All done."
+        done
+
+        usageReport="usage.txt"
+        cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport}
+
+        popd
+
+        # generate report after iterations are over
+        outImage="${dirName}.png"
+        picasso -file ${dirName} -format png -o ${outImage}
+
+    done
+done

test/benchmark/tests/concurrency-level/sample.js

@@ -0,0 +1,10 @@
+import http from "k6/http";
+import { check } from "k6";
+
+export default function() {
+    let params = { timeout: 30 }
+    let res = http.get(`${__ENV.FN_ENDPOINT}`)
+    check(res, {
+        "status is 200": (r) => r.status === 200
+    });
+};

test/benchmark/tests/package-size/run.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ROOT=$(dirname $0)/../../../..
+
+for executorType in poolmgr newdeploy
+do
+    for packagesize in 0 1 5 10 15 20
+    do
+
+        testDuration="5"
+        dirName="package-size-${packagesize}-executor-${executorType}"
+
+        # remove old data
+        rm -rf ${dirName}
+        mkdir ${dirName}
+        pushd ${dirName}
+
+        # run multiple iterations to reduce impact of imbalance of pod distribution.
+        for iteration in {1..10}
+        do
+
+            # Create a hello world function in nodejs, test it with an http trigger
+            echo "Pre-test cleanup"
+            fission env delete --name python || true
+
+            echo "Creating python env"
+            # Use short grace period time to speed up resource recycle time
+            # Use high min/max CPU so that K8S will distribute pod in different nodes
+
+            version=2
+
+            if [[ "${packagesize}" == "0" ]]
+            then
+                version=1
+            fi
+
+            fission env create --name python --version ${version} --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256
+
+            trap "fission env delete --name python" EXIT
+
+            sleep 30
+
+            fn=python-hello-$(date +%s)
+
+            pkgName=""
+
+            if [[ "${packagesize}" == "0" ]]
+            then
+                echo "Creating function"
+                fission fn create --name $fn --env python --code ../../../assets/hello.py --executortype ${executorType} --minscale 3 --maxscale 3
+            else
+                echo "Creating package"
+                rm -rf pkg.zip pkg/ || true
+                mkdir pkg
+                cp ../../../assets/hello.py pkg/hello.py
+
+                # Create empty file with give size to simulate different size of package
+                truncate -s ${packagesize}MiB pkg/foo
+
+                zip -jr pkg.zip pkg/
+                pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2)
+
+                echo "Creating function"
+                fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3
+            fi
+
+            echo "Creating route"
+            fission route create --function $fn --url /$fn --method GET
+
+            echo "Waiting for router to catch up"
+            sleep 5
+
+            fnEndpoint="http://$FISSION_ROUTER/$fn"
+            js="sample.js"
+            rawFile="raw-${iteration}.json"
+            rawUsageReport="raw-usage.txt"
+
+            k6 run \
+                -e FN_ENDPOINT="${fnEndpoint}" \
+                --duration "${testDuration}s" \
+                --rps 1 \
+                --vus 1 \
+                --no-connection-reuse \
+                --out json="${rawFile}" \
+                --summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \
+                ../${js} >> ${rawUsageReport}
+
+            echo "Clean up"
+            fission fn delete --name ${fn}
+            fission env delete --name python
+            fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name
+
+            if [[ ! -z "${pkgName}" ]]
+            then
+                fission pkg delete --name ${pkgName} || true
+                rm -rf pkg.zip pkg
+            fi
+
+            kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
+            kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
+
+            echo "All done."
+        done
+
+        usageReport="usage.txt"
+        cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport}
+
+        popd
+
+        # generate report after iterations are over
+        outImage="${dirName}.png"
+        picasso -file ${dirName} -format png -o ${outImage}
+
+    done
+done

test/benchmark/tests/package-size/sample.js

@@ -0,0 +1,10 @@
+import http from "k6/http";
+import { check } from "k6";
+
+export default function() {
+    let params = { timeout: 30 }
+    let res = http.get(`${__ENV.FN_ENDPOINT}`)
+    check(res, {
+        "status is 200": (r) => r.status === 200
+    });
+};