Add benchmark script (#666)
This commit is contained in:
Ta-Ching Chen
+2
-2
@@ -18,8 +18,8 @@ else
|
||||
fi
|
||||
fi
|
||||
|
||||
go test -v -i $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go')
|
||||
go test -v -i $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v 'benchmark')
|
||||
|
||||
# The executor unit test only works with NodePort-type services for
|
||||
# now. So disable it for our travis ci tests.
|
||||
go test -v $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v executor)
|
||||
go test -v $(go list ./... | grep -v '/vendor/' | grep -v 'examples/go' | grep -v executor | grep -v 'benchmark')
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
FROM golang:1.10.1 AS go-builder
|
||||
WORKDIR /go
|
||||
RUN go get github.com/wcharczuk/go-chart
|
||||
COPY picasso.go /go
|
||||
RUN CGO_ENABLE=0 GOOS=linux GOARCH=amd64 go build -o picasso .
|
||||
|
||||
FROM loadimpact/k6
|
||||
WORKDIR /fission-bench
|
||||
COPY --from=go-builder /go/picasso /usr/local/bin/picasso
|
||||
RUN apk --update add --no-cache bash curl
|
||||
RUN curl -Lo fission https://github.com/fission/fission/releases/download/$(curl --silent "https://api.github.com/repos/fission/fission/releases/latest" | grep "tag_name" |sed -E 's/.*"([^"]+)".*/\1/')/fission-cli-linux && chmod +x fission && mv fission /usr/local/bin/
|
||||
|
||||
ENTRYPOINT ["sh"]
|
||||
@@ -0,0 +1,2 @@
|
||||
def main():
|
||||
return "Hello, world!\n"
|
||||
@@ -0,0 +1,249 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/wcharczuk/go-chart"
|
||||
)
|
||||
|
||||
var (
|
||||
title *string
|
||||
file *string
|
||||
outputFile *string
|
||||
outputFormat *string
|
||||
)
|
||||
|
||||
const (
|
||||
PNG = "png"
|
||||
SVG = "svg"
|
||||
)
|
||||
|
||||
type (
|
||||
Tags struct {
|
||||
Group string `json:"group"`
|
||||
Iter string `json:"iter"`
|
||||
Method string `json:"method"`
|
||||
Name string `json:"name"`
|
||||
Proto string `json:"proto"`
|
||||
Status string `json:"status"`
|
||||
URL string `json:"url"`
|
||||
Vu string `json:"vu"`
|
||||
}
|
||||
|
||||
Data struct {
|
||||
Time time.Time `json:"time"`
|
||||
Value float64 `json:"value"`
|
||||
Tags Tags `json:"tags"`
|
||||
}
|
||||
|
||||
MetricPoint struct {
|
||||
Type string `json:"type"`
|
||||
Data Data `json:"data"`
|
||||
Metric string `json:"metric"`
|
||||
}
|
||||
)
|
||||
|
||||
func init() {
|
||||
title = flag.String("title", "Fission Benchmark", "Chart title")
|
||||
file = flag.String("file", "", "Metric json file")
|
||||
outputFile = flag.String("o", "chart.png", "Output file name")
|
||||
outputFormat = flag.String("format", PNG, "Format of output file (png or svg)")
|
||||
flag.Parse()
|
||||
}
|
||||
|
||||
func generateContinuousSeries(file string) chart.Series {
|
||||
f, err := os.OpenFile(file, os.O_RDONLY, 0644)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to open file metric point: %v", err)
|
||||
return nil
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
reader := bufio.NewReader(f)
|
||||
|
||||
var points []*MetricPoint
|
||||
var xVals []float64
|
||||
var yVals []float64
|
||||
|
||||
var initTime *time.Time
|
||||
|
||||
for {
|
||||
l, _, err := reader.ReadLine()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
|
||||
point := &MetricPoint{}
|
||||
err = json.Unmarshal(l, point)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to parse metric point: %v -> %v", err, string(l))
|
||||
return nil
|
||||
}
|
||||
|
||||
if point.Type != "Point" || point.Metric != "http_req_duration" {
|
||||
continue
|
||||
}
|
||||
|
||||
points = append(points, point)
|
||||
|
||||
if initTime == nil {
|
||||
initTime = &point.Data.Time
|
||||
}
|
||||
|
||||
timeSinceStart := point.Data.Time.Sub(*initTime).Seconds()
|
||||
xVals = append(xVals, timeSinceStart)
|
||||
yVals = append(yVals, point.Data.Value)
|
||||
}
|
||||
|
||||
return chart.ContinuousSeries{
|
||||
Style: chart.Style{
|
||||
Show: true,
|
||||
StrokeColor: chart.GetDefaultColor(0).WithAlpha(64),
|
||||
FillColor: chart.GetDefaultColor(0).WithAlpha(64),
|
||||
},
|
||||
XValues: xVals,
|
||||
YValues: yVals,
|
||||
}
|
||||
}
|
||||
|
||||
func generateChart(title string, file string, format chart.RendererProvider, series []chart.Series) error {
|
||||
if series == nil {
|
||||
return errors.New("Series cannot be nil")
|
||||
}
|
||||
|
||||
cs := chart.ConcatSeries(series)
|
||||
|
||||
graph := chart.Chart{
|
||||
Title: title,
|
||||
TitleStyle: chart.StyleShow(),
|
||||
Background: chart.Style{
|
||||
Padding: chart.Box{
|
||||
Top: 50,
|
||||
Left: 25,
|
||||
Right: 25,
|
||||
Bottom: 10,
|
||||
},
|
||||
},
|
||||
XAxis: chart.XAxis{
|
||||
Name: "Time (s)",
|
||||
NameStyle: chart.StyleShow(),
|
||||
Style: chart.StyleShow(),
|
||||
Range: &chart.ContinuousRange{
|
||||
Min: 0,
|
||||
},
|
||||
ValueFormatter: func(v interface{}) string {
|
||||
return fmt.Sprintf("%.2f s", v.(float64))
|
||||
},
|
||||
},
|
||||
YAxis: chart.YAxis{
|
||||
Name: "Response Time (ms)",
|
||||
NameStyle: chart.StyleShow(),
|
||||
Style: chart.StyleShow(),
|
||||
Range: &chart.ContinuousRange{
|
||||
Min: 0,
|
||||
},
|
||||
ValueFormatter: func(v interface{}) string {
|
||||
return fmt.Sprintf("%d ms", int(v.(float64)))
|
||||
},
|
||||
},
|
||||
Series: cs,
|
||||
}
|
||||
|
||||
buffer := bytes.NewBuffer([]byte{})
|
||||
err := graph.Render(format, buffer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(file, buffer.Bytes(), 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func listJsonFiles(path string) ([]string, error) {
|
||||
fi, err := os.Stat(*file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if fi.Mode().IsRegular() {
|
||||
return []string{path}, nil
|
||||
}
|
||||
|
||||
var files []string
|
||||
|
||||
err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if strings.HasSuffix(path, ".json") && !info.IsDir() {
|
||||
files = append(files, path)
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
if file == nil || len(*file) == 0 {
|
||||
fmt.Println("Please provide metric json file name")
|
||||
return
|
||||
}
|
||||
|
||||
var format chart.RendererProvider
|
||||
|
||||
if outputFormat == nil {
|
||||
format = chart.PNG
|
||||
} else {
|
||||
switch strings.ToLower(*outputFormat) {
|
||||
case PNG:
|
||||
format = chart.PNG
|
||||
case SVG:
|
||||
format = chart.SVG
|
||||
default:
|
||||
fmt.Println("Unknown format, use png as output format")
|
||||
*outputFormat = PNG
|
||||
}
|
||||
}
|
||||
|
||||
if len(*outputFile) == 0 {
|
||||
fmt.Println("Please output chart png file name")
|
||||
return
|
||||
}
|
||||
|
||||
var series []chart.Series
|
||||
|
||||
files, err := listJsonFiles(*file)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to get file information: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
series = append(series, generateContinuousSeries(f))
|
||||
}
|
||||
|
||||
err = generateChart(*title, *outputFile, format, series)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to generate chart: %v\n", err)
|
||||
}
|
||||
}
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=$(dirname $0)/../../../..
|
||||
|
||||
for executorType in poolmgr newdeploy
|
||||
do
|
||||
dirName="burst-load-executor-${executorType}"
|
||||
|
||||
# remove old data
|
||||
rm -rf ${dirName}
|
||||
mkdir ${dirName}
|
||||
pushd ${dirName}
|
||||
|
||||
# run multiple iterations to reduce impact of imbalance of pod distribution.
|
||||
for iteration in {1..10}
|
||||
do
|
||||
|
||||
# Create a hello world function in nodejs, test it with an http trigger
|
||||
echo "Pre-test cleanup"
|
||||
fission env delete --name python || true
|
||||
|
||||
echo "Creating python env"
|
||||
# Use short grace period time to speed up resource recycle time
|
||||
# Use high min/max CPU so that K8S will distribute pod in different nodes
|
||||
fission env create --name python --version 2 --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256
|
||||
trap "fission env delete --name python" EXIT
|
||||
|
||||
sleep 30
|
||||
|
||||
fn=python-hello-$(date +%s)
|
||||
|
||||
echo "Creating package"
|
||||
rm -rf pkg.zip pkg/ || true
|
||||
mkdir pkg
|
||||
cp ../../../assets/hello.py pkg/hello.py
|
||||
|
||||
zip -jr pkg.zip pkg/
|
||||
pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2)
|
||||
|
||||
echo "Creating function"
|
||||
fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3
|
||||
|
||||
echo "Creating route"
|
||||
fission route create --function $fn --url /$fn --method GET
|
||||
|
||||
echo "Waiting for router to catch up"
|
||||
sleep 5
|
||||
|
||||
fnEndpoint="http://$FISSION_ROUTER/$fn"
|
||||
js="sample.js"
|
||||
rawFile="raw-${iteration}.json"
|
||||
rawUsageReport="raw-usage.txt"
|
||||
|
||||
# Stage 1: 15s, 10 vus
|
||||
# Stage 2: 45s, 500 vus
|
||||
# Please check sample.js for more detail.
|
||||
k6 run \
|
||||
-e FN_ENDPOINT="${fnEndpoint}" \
|
||||
--no-connection-reuse \
|
||||
--out json="${rawFile}" \
|
||||
--summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \
|
||||
../${js} >> ${rawUsageReport}
|
||||
|
||||
echo "Clean up"
|
||||
fission fn delete --name ${fn}
|
||||
fission env delete --name python
|
||||
fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name
|
||||
fission pkg delete --name ${pkgName}
|
||||
rm -rf pkg.zip pkg
|
||||
|
||||
kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
|
||||
kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
|
||||
|
||||
echo "All done."
|
||||
done
|
||||
|
||||
usageReport="usage.txt"
|
||||
cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport}
|
||||
|
||||
popd
|
||||
|
||||
# generate report after iterations are over
|
||||
outImage="${dirName}.png"
|
||||
picasso -file ${dirName} -format png -o ${outImage}
|
||||
|
||||
done
|
||||
@@ -0,0 +1,17 @@
|
||||
import http from "k6/http";
|
||||
import { check } from "k6";
|
||||
|
||||
export let options = {
|
||||
stages: [
|
||||
{ duration: "15s", target: 10 },
|
||||
{ duration: "45s", target: 500 },
|
||||
]
|
||||
};
|
||||
|
||||
export default function() {
|
||||
let params = { timeout: 30 }
|
||||
let res = http.get(`${__ENV.FN_ENDPOINT}`)
|
||||
check(res, {
|
||||
"status is 200": (r) => r.status === 200
|
||||
});
|
||||
};
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=$(dirname $0)/../../../..
|
||||
|
||||
for executorType in poolmgr newdeploy
|
||||
do
|
||||
for concurrency in 100 250 500 750 1000
|
||||
do
|
||||
|
||||
testDuration="60"
|
||||
dirName="concurrency-${concurrency}-executor-${executorType}"
|
||||
|
||||
# remove old data
|
||||
rm -rf ${dirName}
|
||||
mkdir ${dirName}
|
||||
pushd ${dirName}
|
||||
|
||||
# run multiple iterations to reduce impact of imbalance of pod distribution.
|
||||
for iteration in {1..10}
|
||||
do
|
||||
|
||||
# Create a hello world function in nodejs, test it with an http trigger
|
||||
echo "Pre-test cleanup"
|
||||
fission env delete --name python || true
|
||||
|
||||
echo "Creating python env"
|
||||
# Use short grace period time to speed up resource recycle time
|
||||
# Use high min/max CPU so that K8S will distribute pod in different nodes
|
||||
fission env create --name python --version 2 --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256
|
||||
trap "fission env delete --name python" EXIT
|
||||
|
||||
sleep 30
|
||||
|
||||
fn=python-hello-$(date +%s)
|
||||
|
||||
echo "Creating package"
|
||||
rm -rf pkg.zip pkg/ || true
|
||||
mkdir pkg
|
||||
cp ../../../assets/hello.py pkg/hello.py
|
||||
|
||||
zip -jr pkg.zip pkg/
|
||||
pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2)
|
||||
|
||||
echo "Creating function"
|
||||
fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3
|
||||
|
||||
echo "Creating route"
|
||||
fission route create --function $fn --url /$fn --method GET
|
||||
|
||||
echo "Waiting for router to catch up"
|
||||
sleep 5
|
||||
|
||||
fnEndpoint="http://$FISSION_ROUTER/$fn"
|
||||
js="sample.js"
|
||||
rawFile="raw-${iteration}.json"
|
||||
rawUsageReport="raw-usage.txt"
|
||||
|
||||
k6 run \
|
||||
-e FN_ENDPOINT="${fnEndpoint}" \
|
||||
--duration "${testDuration}s" \
|
||||
--rps ${concurrency} \
|
||||
--vus ${concurrency} \
|
||||
--no-connection-reuse \
|
||||
--out json="${rawFile}" \
|
||||
--summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \
|
||||
../${js} >> ${rawUsageReport}
|
||||
|
||||
echo "Clean up"
|
||||
fission fn delete --name ${fn}
|
||||
fission env delete --name python
|
||||
fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name
|
||||
fission pkg delete --name ${pkgName}
|
||||
rm -rf pkg.zip pkg
|
||||
|
||||
kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
|
||||
kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
|
||||
|
||||
echo "All done."
|
||||
done
|
||||
|
||||
usageReport="usage.txt"
|
||||
cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport}
|
||||
|
||||
popd
|
||||
|
||||
# generate report after iterations are over
|
||||
outImage="${dirName}.png"
|
||||
picasso -file ${dirName} -format png -o ${outImage}
|
||||
|
||||
done
|
||||
done
|
||||
@@ -0,0 +1,10 @@
|
||||
import http from "k6/http";
|
||||
import { check } from "k6";
|
||||
|
||||
export default function() {
|
||||
let params = { timeout: 30 }
|
||||
let res = http.get(`${__ENV.FN_ENDPOINT}`)
|
||||
check(res, {
|
||||
"status is 200": (r) => r.status === 200
|
||||
});
|
||||
};
|
||||
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT=$(dirname $0)/../../../..
|
||||
|
||||
for executorType in poolmgr newdeploy
|
||||
do
|
||||
for packagesize in 0 1 5 10 15 20
|
||||
do
|
||||
|
||||
testDuration="5"
|
||||
dirName="package-size-${packagesize}-executor-${executorType}"
|
||||
|
||||
# remove old data
|
||||
rm -rf ${dirName}
|
||||
mkdir ${dirName}
|
||||
pushd ${dirName}
|
||||
|
||||
# run multiple iterations to reduce impact of imbalance of pod distribution.
|
||||
for iteration in {1..10}
|
||||
do
|
||||
|
||||
# Create a hello world function in nodejs, test it with an http trigger
|
||||
echo "Pre-test cleanup"
|
||||
fission env delete --name python || true
|
||||
|
||||
echo "Creating python env"
|
||||
# Use short grace period time to speed up resource recycle time
|
||||
# Use high min/max CPU so that K8S will distribute pod in different nodes
|
||||
|
||||
version=2
|
||||
|
||||
if [[ "${packagesize}" == "0" ]]
|
||||
then
|
||||
version=1
|
||||
fi
|
||||
|
||||
fission env create --name python --version ${version} --image fission/python-env --period 5 --mincpu 300 --maxcpu 300 --minmemory 256 --maxmemory 256
|
||||
|
||||
trap "fission env delete --name python" EXIT
|
||||
|
||||
sleep 30
|
||||
|
||||
fn=python-hello-$(date +%s)
|
||||
|
||||
pkgName=""
|
||||
|
||||
if [[ "${packagesize}" == "0" ]]
|
||||
then
|
||||
echo "Creating function"
|
||||
fission fn create --name $fn --env python --code ../../../assets/hello.py --executortype ${executorType} --minscale 3 --maxscale 3
|
||||
else
|
||||
echo "Creating package"
|
||||
rm -rf pkg.zip pkg/ || true
|
||||
mkdir pkg
|
||||
cp ../../../assets/hello.py pkg/hello.py
|
||||
|
||||
# Create empty file with give size to simulate different size of package
|
||||
truncate -s ${packagesize}MiB pkg/foo
|
||||
|
||||
zip -jr pkg.zip pkg/
|
||||
pkgName=$(fission pkg create --env python --deploy pkg.zip | cut -d' ' -f 2 | cut -d"'" -f 2)
|
||||
|
||||
echo "Creating function"
|
||||
fission fn create --name $fn --env python --pkg ${pkgName} --entrypoint "hello.main" --executortype ${executorType} --minscale 3 --maxscale 3
|
||||
fi
|
||||
|
||||
echo "Creating route"
|
||||
fission route create --function $fn --url /$fn --method GET
|
||||
|
||||
echo "Waiting for router to catch up"
|
||||
sleep 5
|
||||
|
||||
fnEndpoint="http://$FISSION_ROUTER/$fn"
|
||||
js="sample.js"
|
||||
rawFile="raw-${iteration}.json"
|
||||
rawUsageReport="raw-usage.txt"
|
||||
|
||||
k6 run \
|
||||
-e FN_ENDPOINT="${fnEndpoint}" \
|
||||
--duration "${testDuration}s" \
|
||||
--rps 1 \
|
||||
--vus 1 \
|
||||
--no-connection-reuse \
|
||||
--out json="${rawFile}" \
|
||||
--summary-trend-stats="avg,min,med,max,p(5),p(10),p(15),p(20),p(25),p(30),p(35),p(40),p(45),p(50),p(55),p(60),p(65),p(70),p(75),p(80),p(85),p(90),p(95),p(100)" \
|
||||
../${js} >> ${rawUsageReport}
|
||||
|
||||
echo "Clean up"
|
||||
fission fn delete --name ${fn}
|
||||
fission env delete --name python
|
||||
fission route list| grep ${fn}| awk '{print $1}'| xargs fission route delete --name
|
||||
|
||||
if [[ ! -z "${pkgName}" ]]
|
||||
then
|
||||
fission pkg delete --name ${pkgName} || true
|
||||
rm -rf pkg.zip pkg
|
||||
fi
|
||||
|
||||
kubectl -n fission-function get deploy -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
|
||||
kubectl -n fission-function get pod -o name|xargs -I@ bash -c "kubectl -n fission-function delete @" || true
|
||||
|
||||
echo "All done."
|
||||
done
|
||||
|
||||
usageReport="usage.txt"
|
||||
cat ${rawUsageReport}| grep "http_req_duration"| cut -f2 -d':' > ${usageReport}
|
||||
|
||||
popd
|
||||
|
||||
# generate report after iterations are over
|
||||
outImage="${dirName}.png"
|
||||
picasso -file ${dirName} -format png -o ${outImage}
|
||||
|
||||
done
|
||||
done
|
||||
@@ -0,0 +1,10 @@
|
||||
import http from "k6/http";
|
||||
import { check } from "k6";
|
||||
|
||||
export default function() {
|
||||
let params = { timeout: 30 }
|
||||
let res = http.get(`${__ENV.FN_ENDPOINT}`)
|
||||
check(res, {
|
||||
"status is 200": (r) => r.status === 200
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user