Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add experimental environment: tensorflow-serving (#1212)

Browse Source
This commit is contained in:
Ta-Ching Chen
2019-07-02 15:43:55 +08:00
committed by GitHub
parent 2ba66afb9f
commit b229d6e1b0
13 changed files with 408 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
environments/tensorflow-serving/Dockerfile
+23
View File
@@ -0,0 +1,23 @@
ARG GO_VERSION=1.9.2
FROM tensorflow/serving as serving
RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/*
FROM golang:${GO_VERSION} AS builder
ENV GOPATH /usr
ENV APP ${GOPATH}/src/github.com/fission/fission/environments/tensorflow-serving
WORKDIR ${APP}
ADD server.go ${APP}
RUN go get
RUN go build -a -o /server server.go
FROM serving
WORKDIR /
COPY --from=builder /server /
ENTRYPOINT ["/server"]
EXPOSE 8888
environments/tensorflow-serving/README.md
+29
View File
@@ -0,0 +1,29 @@
# Fission: Tensorflow Serving Environment
This is the Tensorflow Serving environment for Fission.
It's a Docker image containing a Go runtime, along with a tensorflow serving service.
## How it works
Tensorflow Serving is an serving service that supports both RESTful API and gRPC endpoints. In current implementation,
Go server launches `tensorflow_model_server` to load in model during specialization. As long as the Go server receives
requests from router it creates a reverse proxy that connects to RESTful API endpoint exposed by tensorflow_model_server
and get response from the upstream server for user.
## Build this image
```
docker build -t USER/tensorflow-serving . && docker push USER/tensorflow-serving
```
## Using the image in fission
You can add this customized image to fission with "fission env create":
```
fission env create --name tensorflow --image USER/tensorflow-serving --version 2
```
After this, fission functions that have the env parameter set to the
same environment name as this command will use this environment.
environments/tensorflow-serving/server.go
+205
View File
@@ -0,0 +1,205 @@
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"net/http/httputil"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/pkg/errors"
"go.uber.org/zap"
)
const (
PortgRPC = 8500
PortRestAPI = 8501
)
var (
specialized = false
// for tensorflow serving to use
MODEL_NAME = ""
)
type (
FunctionLoadRequest struct {
// FilePath is an absolute filesystem path to the
// function. What exactly is stored here is
// env-specific. Optional.
FilePath string `json:"filepath"`
// FunctionName has an environment-specific meaning;
// usually, it defines a function within a module
// containing multiple functions. Optional; default is
// environment-specific.
FunctionName string `json:"functionName"`
// URL to expose this function at. Optional; defaults
// to "/".
URL string `json:"url"`
}
)
func specializeHandler(logger *zap.Logger) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
logger.Error("v1 interface is not implemented")
w.WriteHeader(http.StatusNotImplemented)
}
}
func specializeHandlerV2(logger *zap.Logger) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
if specialized {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Not a generic container"))
return
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
logger.Error("error reading request body", zap.Error(err))
w.WriteHeader(http.StatusInternalServerError)
return
}
var loadreq FunctionLoadRequest
err = json.Unmarshal(body, &loadreq)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
// To ensure we load model from the expected path
basePath := fmt.Sprintf("%v/%v", loadreq.FilePath, loadreq.FunctionName)
basePath, err = filepath.Abs(basePath)
if err != nil {
msg := "error getting absolute path of model"
logger.Error(msg, zap.Error(err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
} else if !strings.HasPrefix(basePath, loadreq.FilePath) {
msg := "incorrect model base path"
logger.Error(msg, zap.String("model_base_path", basePath))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(msg))
return
}
_, err = os.Stat(basePath)
if err != nil {
msg := "error checking model status"
logger.Error(msg, zap.Error(err))
w.WriteHeader(http.StatusInternalServerError)
return
}
// get directory name that holds model
MODEL_NAME = filepath.Base(basePath)
argModelBasePath := fmt.Sprintf("--model_base_path=%v", basePath)
argModelName := fmt.Sprintf("--model_name=%v", MODEL_NAME)
argPortgRPC := fmt.Sprintf("--port=%v", PortgRPC)
argPortREST := fmt.Sprintf("--rest_api_port=%v", PortRestAPI)
logger.Info(fmt.Sprintf("specializing: %v %v", loadreq.FunctionName, loadreq.FilePath))
// Future: could be improved by keeping subprocess open while environment is specialized
cmd := exec.Command("tensorflow_model_server",
argPortgRPC, argPortREST, argModelName, argModelBasePath)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Start()
if err != nil {
msg := "error starting tensorflow serving"
logger.Error(msg, zap.Error(err))
err = errors.Wrap(err, msg)
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
go func() {
err = cmd.Wait()
if err != nil {
logger.Fatal("error running tensorflow serving", zap.Error(err))
}
}()
t := time.Now()
retryInterval := 50 * time.Millisecond
// tensorflow serving takes some time to load model
// into memory, keep retrying until it starts REST api server.
for {
if time.Since(t) > 30*time.Second {
w.WriteHeader(http.StatusGatewayTimeout)
return
}
conn, err := net.Dial("tcp", "localhost:8501")
if err == nil {
conn.Close()
break
} else {
logger.Info(fmt.Sprintf("waiting for tensorflow serving to be ready: %v", err.Error()))
time.Sleep(retryInterval)
retryInterval = retryInterval * 2
}
}
specialized = true
logger.Info("done")
}
}
func readinessProbeHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
http.HandleFunc("/healthz", readinessProbeHandler)
http.HandleFunc("/specialize", specializeHandler(logger.Named("specialize_handler")))
http.HandleFunc("/v2/specialize", specializeHandlerV2(logger.Named("specialize_v2_handler")))
// Generic route -- all http requests go to the user function.
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if !specialized {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Generic container: no requests supported"))
return
}
// TODO: replace it with gRPC (https://gist.github.com/mauri870/1f953a183ee6c186e70a0a72e78b088c)
// set up proxy server director
director := func(req *http.Request) {
req.URL.Scheme = "http"
req.URL.Host = "localhost:8501"
req.URL.Path = fmt.Sprintf("/v1/models/%v:predict", MODEL_NAME)
}
proxy := &httputil.ReverseProxy{
Director: director,
}
proxy.ServeHTTP(w, r)
})
logger.Info("listening on 8888 ...")
http.ListenAndServe(":8888", nil)
}
examples/tensorflow-serving/README.md
+23
View File
@@ -0,0 +1,23 @@
# Tensorflow Serving Environment Example
## Create Environment
```bash
$ fission env create --name tensorflow --image fission/tensorflow-serving --version 2
```
## Create Package
```bash
$ zip -r half_plus_two.zip ./half_plus_two
$ fission pkg create --env tensorflow --deploy half_plus_two.zip
```
## Create Function
Here, the `--entrypoint` represents the name of top directory contains trained model.
```bash
$ fission fn create --name t1 --pkg <pkg name> --env tensorflow --entrypoint "half_plus_two"
$ fission fn test --name t1 --body '{"instances": [1.0, 2.0, 0.0]}' --method POST
```
examples/tensorflow-serving/half_plus_two/00000123/saved_model.pb
BIN
View File
Binary file not shown.
examples/tensorflow-serving/half_plus_two/00000123/variables/variables.data-00000-of-00001
BIN
View File
Binary file not shown.
examples/tensorflow-serving/half_plus_two/00000123/variables/variables.index
BIN
View File
Binary file not shown.
hack/release-build.sh
+14 -13
View File
@@ -131,19 +131,20 @@ build_all_envs() {
local version=$1
# call with version, env dir, image name base, image name variant
build_env_image "$version" "nodejs" "node-env" ""
build_env_image "$version" "nodejs" "node-env" "debian"
build_env_image "$version" "binary" "binary-env" ""
build_env_image "$version" "dotnet" "dotnet-env" ""
build_env_image "$version" "dotnet20" "dotnet20-env" ""
build_env_image "$version" "go" "go-env" ""
build_env_image "$version" "go" "go-env" "1.11.4"
build_env_image "$version" "perl" "perl-env" ""
build_env_image "$version" "php7" "php-env" ""
build_env_image "$version" "python" "python-env" ""
build_env_image "$version" "python" "python-env" "2.7"
build_env_image "$version" "ruby" "ruby-env" ""
build_env_image "$version" "jvm" "jvm-env" ""
build_env_image "$version" "nodejs" "node-env" ""
build_env_image "$version" "nodejs" "node-env" "debian"
build_env_image "$version" "binary" "binary-env" ""
build_env_image "$version" "dotnet" "dotnet-env" ""
build_env_image "$version" "dotnet20" "dotnet20-env" ""
build_env_image "$version" "go" "go-env" ""
build_env_image "$version" "go" "go-env" "1.11.4"
build_env_image "$version" "perl" "perl-env" ""
build_env_image "$version" "php7" "php-env" ""
build_env_image "$version" "python" "python-env" ""
build_env_image "$version" "python" "python-env" "2.7"
build_env_image "$version" "ruby" "ruby-env" ""
build_env_image "$version" "jvm" "jvm-env" ""
build_env_image "$version" "tensorflow-serving" "tensorflow-serving-env" ""
}
build_env_builder_image() {
hack/release.sh
+14 -13
View File
@@ -71,19 +71,20 @@ push_all_envs() {
local version=$1
# call with version, env dir, image name base, image name variant
push_env_image "$version" "nodejs" "node-env" ""
push_env_image "$version" "nodejs" "node-env" "debian"
push_env_image "$version" "binary" "binary-env" ""
push_env_image "$version" "dotnet" "dotnet-env" ""
push_env_image "$version" "dotnet20" "dotnet20-env" ""
push_env_image "$version" "go" "go-env" ""
push_env_image "$version" "go" "go-env" "1.11.4"
push_env_image "$version" "perl" "perl-env" ""
push_env_image "$version" "php7" "php-env" ""
push_env_image "$version" "python" "python-env" ""
push_env_image "$version" "python" "python-env" "2.7"
push_env_image "$version" "ruby" "ruby-env" ""
push_env_image "$version" "jvm" "jvm-env" ""
push_env_image "$version" "nodejs" "node-env" ""
push_env_image "$version" "nodejs" "node-env" "debian"
push_env_image "$version" "binary" "binary-env" ""
push_env_image "$version" "dotnet" "dotnet-env" ""
push_env_image "$version" "dotnet20" "dotnet20-env" ""
push_env_image "$version" "go" "go-env" ""
push_env_image "$version" "go" "go-env" "1.11.4"
push_env_image "$version" "perl" "perl-env" ""
push_env_image "$version" "php7" "php-env" ""
push_env_image "$version" "python" "python-env" ""
push_env_image "$version" "python" "python-env" "2.7"
push_env_image "$version" "ruby" "ruby-env" ""
push_env_image "$version" "jvm" "jvm-env" ""
push_env_image "$version" "tensorflow-serving" "tensorflow-serving-env" ""
}
push_env_builder_image() {
test/build_and_test.sh
+1
View File
@@ -40,6 +40,7 @@ build_and_push_builder $BUILDER_IMAGE:$TAG
build_and_push_env_runtime python $REPO/python-env:$TAG
build_and_push_env_runtime jvm $REPO/jvm-env:$TAG
build_and_push_env_runtime go $REPO/go-env:$TAG
build_and_push_env_runtime tensorflow-serving $REPO/tensorflow-serving-env:$TAG
build_and_push_env_builder python $REPO/python-env-builder:$TAG $BUILDER_IMAGE:$TAG
build_and_push_env_builder jvm $REPO/jvm-env-builder:$TAG $BUILDER_IMAGE:$TAG
test/test_utils.sh
+5 -3
View File
@@ -441,6 +441,7 @@ run_all_tests() {
export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:test
export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:test
export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:test
export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:test
set +e
export TIMEOUT=900 # 15 minutes per test
@@ -448,6 +449,7 @@ run_all_tests() {
# run tests without newdeploy in parallel.
export JOBS=6
$ROOT/test/run_test.sh \
$ROOT/test/tests/test_canary.sh \
$ROOT/test/tests/mqtrigger/kafka/test_kafka.sh \
$ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \
$ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh \
@@ -458,7 +460,6 @@ run_all_tests() {
$ROOT/test/tests/test_archive_pruner.sh \
$ROOT/test/tests/test_backend_poolmgr.sh \
$ROOT/test/tests/test_buildermgr.sh \
$ROOT/test/tests/test_canary.sh \
$ROOT/test/tests/test_env_vars.sh \
$ROOT/test/tests/test_environments/test_python_env.sh \
$ROOT/test/tests/test_fn_update/test_idle_objects_reaper.sh \
@@ -473,14 +474,15 @@ run_all_tests() {
$ROOT/test/tests/test_router_cache_invalidation.sh \
$ROOT/test/tests/test_specs/test_spec.sh \
$ROOT/test/tests/test_specs/test_spec_multifile.sh \
$ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh
$ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh \
$ROOT/test/tests/test_environments/test_tensorflow_serving_env.sh \
$ROOT/test/tests/test_environments/test_go_env.sh
FAILURES=$?
# FIXME: run tests with newdeploy one by one.
export JOBS=1
$ROOT/test/run_test.sh \
$ROOT/test/tests/test_backend_newdeploy.sh \
$ROOT/test/tests/test_environments/test_go_env.sh \
$ROOT/test/tests/test_environments/test_java_builder.sh \
$ROOT/test/tests/test_environments/test_java_env.sh \
$ROOT/test/tests/test_fn_update/test_configmap_update.sh \
test/tests/test_environments/test_tensorflow_serving_env.sh
Executable
+61
View File
@@ -0,0 +1,61 @@
#!/bin/bash
set -euo pipefail
source $(dirname $0)/../../utils.sh
TEST_ID=$(generate_test_id)
echo "TEST_ID = $TEST_ID"
tmp_dir="/tmp/test-$TEST_ID"
mkdir -p $tmp_dir
ROOT=$(dirname $0)/../../..
cleanup() {
clean_resource_by_id $TEST_ID
rm -rf $tmp_dir
}
if [ -z "${TEST_NOCLEANUP:-}" ]; then
trap cleanup EXIT
else
log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards."
fi
env=ts-$TEST_ID
fn_poolmgr=hello-ts-poolmgr-$TEST_ID
fn_nd=hello-ts-nd-$TEST_ID
cd $ROOT/examples/tensorflow-serving
log "Creating environment for Tensorflow Serving"
fission env create --name $env --image $TS_RUNTIME_IMAGE --version 2 --period 5
zip -r half_plus_two.zip ./half_plus_two
pkgName=$(fission package create --deploy half_plus_two.zip --env $env| cut -f2 -d' '| tr -d \')
# wait for build to finish at most 90s
timeout 90 bash -c "waitBuild $pkgName"
log "Creating pool manager & new deployment function for Tensorflow Serving"
fission fn create --name $fn_poolmgr --env $env --pkg $pkgName --entrypoint "half_plus_two"
fission fn create --name $fn_nd --env $env --pkg $pkgName --entrypoint "half_plus_two" --executortype newdeploy
log "Creating route for new deployment function"
fission route create --function $fn_poolmgr --url /$fn_poolmgr --method POST
fission route create --function $fn_nd --url /$fn_nd --method POST
log "Waiting for router & pools to catch up"
sleep 5
body='{\"instances\": [1.0, 2.0, 5.0]}'
expect='\"predictions\": \[2.5, 3.0, 4.5'
log "Testing pool manager function"
timeout 60 bash -c "test_post_route $fn_poolmgr \"$body\" \"$expect\""
log "Testing new deployment function"
timeout 60 bash -c "test_post_route $fn_nd \"$body\" \"$expect\""
log "Test PASSED"
test/utils.sh
+33 -1
View File
@@ -71,6 +71,35 @@ test_fn() {
}
export -f test_fn
test_post_route() {
# Doing an HTTP POST on the function's route
# Checking for valid response
url="http://$FISSION_ROUTER/$1"
body=$2
expect=$3
set +e
while true; do
log "test_post_route: call curl"
resp=$(curl --silent --show-error -d "$body" -X POST "$url")
status_code=$?
if [ $status_code -ne 0 ]; then
log "test_post_route: curl failed ($status_code). Retrying ..."
sleep 1
continue
fi
if ! (echo $resp | grep "$expect" > /dev/null); then
log "test_post_route: resp = '$resp' expect = '$expect'"
log "test_post_route: expected string not found. Retrying ..."
sleep 1
continue
fi
break
done
set -e
}
export -f test_post_route
wait_for_builder() {
env=$1
JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}'
@@ -108,7 +137,9 @@ export -f waitBuild
export FISSION_NAMESPACE=${FISSION_NAMESPACE:-fission}
export FUNCTION_NAMESPACE=${FUNCTION_NAMESPACE:-fission-function}
export FISSION_ROUTER=$(kubectl -n $FISSION_NAMESPACE get svc router -o jsonpath='{...ip}')
router=$(kubectl -n $FISSION_NAMESPACE get svc router -o jsonpath='{...ip}')
export FISSION_ROUTER=${FISSION_ROUTER:-$router}
export FISSION_NATS_STREAMING_URL="http://defaultFissionAuthToken@$(kubectl -n $FISSION_NAMESPACE get svc nats-streaming -o jsonpath='{...ip}:{.spec.ports[0].port}')"
## Parameters used by some specific test cases
@@ -119,4 +150,5 @@ export GO_BUILDER_IMAGE=${GO_BUILDER_IMAGE:-fission/go-builder}
export JVM_RUNTIME_IMAGE=${JVM_RUNTIME_IMAGE:-fission/jvm-env}
export JVM_BUILDER_IMAGE=${JVM_BUILDER_IMAGE:-fission/jvm-builder}
export NODE_RUNTIME_IMAGE=${NODE_RUNTIME_IMAGE:-fission/node-env}
export TS_RUNTIME_IMAGE=${TS_RUNTIME_IMAGE:-fission/tensorflow-serving-env}