diff --git a/environments/tensorflow-serving/Dockerfile b/environments/tensorflow-serving/Dockerfile new file mode 100644 index 00000000..ade52016 --- /dev/null +++ b/environments/tensorflow-serving/Dockerfile @@ -0,0 +1,23 @@ +ARG GO_VERSION=1.9.2 + +FROM tensorflow/serving as serving +RUN apt update && apt install -y ca-certificates && rm -rf /var/lib/apt/lists/* + +FROM golang:${GO_VERSION} AS builder + +ENV GOPATH /usr +ENV APP ${GOPATH}/src/github.com/fission/fission/environments/tensorflow-serving + +WORKDIR ${APP} + +ADD server.go ${APP} + +RUN go get +RUN go build -a -o /server server.go + +FROM serving +WORKDIR / +COPY --from=builder /server / + +ENTRYPOINT ["/server"] +EXPOSE 8888 diff --git a/environments/tensorflow-serving/README.md b/environments/tensorflow-serving/README.md new file mode 100644 index 00000000..a1c4ea1c --- /dev/null +++ b/environments/tensorflow-serving/README.md @@ -0,0 +1,29 @@ +# Fission: Tensorflow Serving Environment + +This is the Tensorflow Serving environment for Fission. + +It's a Docker image containing a Go runtime, along with a tensorflow serving service. + +## How it works + +Tensorflow Serving is an serving service that supports both RESTful API and gRPC endpoints. In current implementation, +Go server launches `tensorflow_model_server` to load in model during specialization. As long as the Go server receives +requests from router it creates a reverse proxy that connects to RESTful API endpoint exposed by tensorflow_model_server +and get response from the upstream server for user. + +## Build this image + +``` +docker build -t USER/tensorflow-serving . && docker push USER/tensorflow-serving +``` + +## Using the image in fission + +You can add this customized image to fission with "fission env create": + +``` +fission env create --name tensorflow --image USER/tensorflow-serving --version 2 +``` + +After this, fission functions that have the env parameter set to the +same environment name as this command will use this environment. diff --git a/environments/tensorflow-serving/server.go b/environments/tensorflow-serving/server.go new file mode 100644 index 00000000..d935476f --- /dev/null +++ b/environments/tensorflow-serving/server.go @@ -0,0 +1,205 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net" + "net/http" + "net/http/httputil" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + "go.uber.org/zap" +) + +const ( + PortgRPC = 8500 + PortRestAPI = 8501 +) + +var ( + specialized = false + + // for tensorflow serving to use + MODEL_NAME = "" +) + +type ( + FunctionLoadRequest struct { + // FilePath is an absolute filesystem path to the + // function. What exactly is stored here is + // env-specific. Optional. + FilePath string `json:"filepath"` + + // FunctionName has an environment-specific meaning; + // usually, it defines a function within a module + // containing multiple functions. Optional; default is + // environment-specific. + FunctionName string `json:"functionName"` + + // URL to expose this function at. Optional; defaults + // to "/". + URL string `json:"url"` + } +) + +func specializeHandler(logger *zap.Logger) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + logger.Error("v1 interface is not implemented") + w.WriteHeader(http.StatusNotImplemented) + } +} + +func specializeHandlerV2(logger *zap.Logger) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + if specialized { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Not a generic container")) + return + } + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + logger.Error("error reading request body", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + var loadreq FunctionLoadRequest + err = json.Unmarshal(body, &loadreq) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + // To ensure we load model from the expected path + basePath := fmt.Sprintf("%v/%v", loadreq.FilePath, loadreq.FunctionName) + basePath, err = filepath.Abs(basePath) + if err != nil { + msg := "error getting absolute path of model" + logger.Error(msg, zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } else if !strings.HasPrefix(basePath, loadreq.FilePath) { + msg := "incorrect model base path" + logger.Error(msg, zap.String("model_base_path", basePath)) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(msg)) + return + } + + _, err = os.Stat(basePath) + if err != nil { + msg := "error checking model status" + logger.Error(msg, zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + + // get directory name that holds model + MODEL_NAME = filepath.Base(basePath) + + argModelBasePath := fmt.Sprintf("--model_base_path=%v", basePath) + argModelName := fmt.Sprintf("--model_name=%v", MODEL_NAME) + argPortgRPC := fmt.Sprintf("--port=%v", PortgRPC) + argPortREST := fmt.Sprintf("--rest_api_port=%v", PortRestAPI) + + logger.Info(fmt.Sprintf("specializing: %v %v", loadreq.FunctionName, loadreq.FilePath)) + + // Future: could be improved by keeping subprocess open while environment is specialized + cmd := exec.Command("tensorflow_model_server", + argPortgRPC, argPortREST, argModelName, argModelBasePath) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err = cmd.Start() + if err != nil { + msg := "error starting tensorflow serving" + logger.Error(msg, zap.Error(err)) + err = errors.Wrap(err, msg) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + + go func() { + err = cmd.Wait() + if err != nil { + logger.Fatal("error running tensorflow serving", zap.Error(err)) + } + }() + + t := time.Now() + retryInterval := 50 * time.Millisecond + + // tensorflow serving takes some time to load model + // into memory, keep retrying until it starts REST api server. + for { + if time.Since(t) > 30*time.Second { + w.WriteHeader(http.StatusGatewayTimeout) + return + } + conn, err := net.Dial("tcp", "localhost:8501") + if err == nil { + conn.Close() + break + } else { + logger.Info(fmt.Sprintf("waiting for tensorflow serving to be ready: %v", err.Error())) + time.Sleep(retryInterval) + retryInterval = retryInterval * 2 + } + } + + specialized = true + + logger.Info("done") + } +} + +func readinessProbeHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +func main() { + logger, err := zap.NewProduction() + if err != nil { + log.Fatalf("can't initialize zap logger: %v", err) + } + defer logger.Sync() + + http.HandleFunc("/healthz", readinessProbeHandler) + http.HandleFunc("/specialize", specializeHandler(logger.Named("specialize_handler"))) + http.HandleFunc("/v2/specialize", specializeHandlerV2(logger.Named("specialize_v2_handler"))) + + // Generic route -- all http requests go to the user function. + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if !specialized { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("Generic container: no requests supported")) + return + } + + // TODO: replace it with gRPC (https://gist.github.com/mauri870/1f953a183ee6c186e70a0a72e78b088c) + // set up proxy server director + director := func(req *http.Request) { + req.URL.Scheme = "http" + req.URL.Host = "localhost:8501" + req.URL.Path = fmt.Sprintf("/v1/models/%v:predict", MODEL_NAME) + } + + proxy := &httputil.ReverseProxy{ + Director: director, + } + proxy.ServeHTTP(w, r) + }) + + logger.Info("listening on 8888 ...") + http.ListenAndServe(":8888", nil) +} diff --git a/examples/tensorflow-serving/README.md b/examples/tensorflow-serving/README.md new file mode 100644 index 00000000..591829f8 --- /dev/null +++ b/examples/tensorflow-serving/README.md @@ -0,0 +1,23 @@ +# Tensorflow Serving Environment Example + +## Create Environment + +```bash +$ fission env create --name tensorflow --image fission/tensorflow-serving --version 2 +``` + +## Create Package + +```bash +$ zip -r half_plus_two.zip ./half_plus_two +$ fission pkg create --env tensorflow --deploy half_plus_two.zip +``` + +## Create Function + +Here, the `--entrypoint` represents the name of top directory contains trained model. + +```bash +$ fission fn create --name t1 --pkg --env tensorflow --entrypoint "half_plus_two" +$ fission fn test --name t1 --body '{"instances": [1.0, 2.0, 0.0]}' --method POST +``` diff --git a/examples/tensorflow-serving/half_plus_two/00000123/saved_model.pb b/examples/tensorflow-serving/half_plus_two/00000123/saved_model.pb new file mode 100644 index 00000000..c79ebe01 Binary files /dev/null and b/examples/tensorflow-serving/half_plus_two/00000123/saved_model.pb differ diff --git a/examples/tensorflow-serving/half_plus_two/00000123/variables/variables.data-00000-of-00001 b/examples/tensorflow-serving/half_plus_two/00000123/variables/variables.data-00000-of-00001 new file mode 100644 index 00000000..15b75d6e Binary files /dev/null and b/examples/tensorflow-serving/half_plus_two/00000123/variables/variables.data-00000-of-00001 differ diff --git a/examples/tensorflow-serving/half_plus_two/00000123/variables/variables.index b/examples/tensorflow-serving/half_plus_two/00000123/variables/variables.index new file mode 100644 index 00000000..7ec9fb4f Binary files /dev/null and b/examples/tensorflow-serving/half_plus_two/00000123/variables/variables.index differ diff --git a/hack/release-build.sh b/hack/release-build.sh index 85fc30e6..b59dba41 100755 --- a/hack/release-build.sh +++ b/hack/release-build.sh @@ -131,19 +131,20 @@ build_all_envs() { local version=$1 # call with version, env dir, image name base, image name variant - build_env_image "$version" "nodejs" "node-env" "" - build_env_image "$version" "nodejs" "node-env" "debian" - build_env_image "$version" "binary" "binary-env" "" - build_env_image "$version" "dotnet" "dotnet-env" "" - build_env_image "$version" "dotnet20" "dotnet20-env" "" - build_env_image "$version" "go" "go-env" "" - build_env_image "$version" "go" "go-env" "1.11.4" - build_env_image "$version" "perl" "perl-env" "" - build_env_image "$version" "php7" "php-env" "" - build_env_image "$version" "python" "python-env" "" - build_env_image "$version" "python" "python-env" "2.7" - build_env_image "$version" "ruby" "ruby-env" "" - build_env_image "$version" "jvm" "jvm-env" "" + build_env_image "$version" "nodejs" "node-env" "" + build_env_image "$version" "nodejs" "node-env" "debian" + build_env_image "$version" "binary" "binary-env" "" + build_env_image "$version" "dotnet" "dotnet-env" "" + build_env_image "$version" "dotnet20" "dotnet20-env" "" + build_env_image "$version" "go" "go-env" "" + build_env_image "$version" "go" "go-env" "1.11.4" + build_env_image "$version" "perl" "perl-env" "" + build_env_image "$version" "php7" "php-env" "" + build_env_image "$version" "python" "python-env" "" + build_env_image "$version" "python" "python-env" "2.7" + build_env_image "$version" "ruby" "ruby-env" "" + build_env_image "$version" "jvm" "jvm-env" "" + build_env_image "$version" "tensorflow-serving" "tensorflow-serving-env" "" } build_env_builder_image() { diff --git a/hack/release.sh b/hack/release.sh index 7aad7419..bbfc7230 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -71,19 +71,20 @@ push_all_envs() { local version=$1 # call with version, env dir, image name base, image name variant - push_env_image "$version" "nodejs" "node-env" "" - push_env_image "$version" "nodejs" "node-env" "debian" - push_env_image "$version" "binary" "binary-env" "" - push_env_image "$version" "dotnet" "dotnet-env" "" - push_env_image "$version" "dotnet20" "dotnet20-env" "" - push_env_image "$version" "go" "go-env" "" - push_env_image "$version" "go" "go-env" "1.11.4" - push_env_image "$version" "perl" "perl-env" "" - push_env_image "$version" "php7" "php-env" "" - push_env_image "$version" "python" "python-env" "" - push_env_image "$version" "python" "python-env" "2.7" - push_env_image "$version" "ruby" "ruby-env" "" - push_env_image "$version" "jvm" "jvm-env" "" + push_env_image "$version" "nodejs" "node-env" "" + push_env_image "$version" "nodejs" "node-env" "debian" + push_env_image "$version" "binary" "binary-env" "" + push_env_image "$version" "dotnet" "dotnet-env" "" + push_env_image "$version" "dotnet20" "dotnet20-env" "" + push_env_image "$version" "go" "go-env" "" + push_env_image "$version" "go" "go-env" "1.11.4" + push_env_image "$version" "perl" "perl-env" "" + push_env_image "$version" "php7" "php-env" "" + push_env_image "$version" "python" "python-env" "" + push_env_image "$version" "python" "python-env" "2.7" + push_env_image "$version" "ruby" "ruby-env" "" + push_env_image "$version" "jvm" "jvm-env" "" + push_env_image "$version" "tensorflow-serving" "tensorflow-serving-env" "" } push_env_builder_image() { diff --git a/test/build_and_test.sh b/test/build_and_test.sh index de681c33..c7f7b3d9 100755 --- a/test/build_and_test.sh +++ b/test/build_and_test.sh @@ -40,6 +40,7 @@ build_and_push_builder $BUILDER_IMAGE:$TAG build_and_push_env_runtime python $REPO/python-env:$TAG build_and_push_env_runtime jvm $REPO/jvm-env:$TAG build_and_push_env_runtime go $REPO/go-env:$TAG +build_and_push_env_runtime tensorflow-serving $REPO/tensorflow-serving-env:$TAG build_and_push_env_builder python $REPO/python-env-builder:$TAG $BUILDER_IMAGE:$TAG build_and_push_env_builder jvm $REPO/jvm-env-builder:$TAG $BUILDER_IMAGE:$TAG diff --git a/test/test_utils.sh b/test/test_utils.sh index f40a27ee..e058d427 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -441,6 +441,7 @@ run_all_tests() { export GO_BUILDER_IMAGE=gcr.io/fission-ci/go-env-builder:test export JVM_RUNTIME_IMAGE=gcr.io/fission-ci/jvm-env:test export JVM_BUILDER_IMAGE=gcr.io/fission-ci/jvm-env-builder:test + export TS_RUNTIME_IMAGE=gcr.io/fission-ci/tensorflow-serving-env:test set +e export TIMEOUT=900 # 15 minutes per test @@ -448,6 +449,7 @@ run_all_tests() { # run tests without newdeploy in parallel. export JOBS=6 $ROOT/test/run_test.sh \ + $ROOT/test/tests/test_canary.sh \ $ROOT/test/tests/mqtrigger/kafka/test_kafka.sh \ $ROOT/test/tests/mqtrigger/nats/test_mqtrigger.sh \ $ROOT/test/tests/mqtrigger/nats/test_mqtrigger_error.sh \ @@ -458,7 +460,6 @@ run_all_tests() { $ROOT/test/tests/test_archive_pruner.sh \ $ROOT/test/tests/test_backend_poolmgr.sh \ $ROOT/test/tests/test_buildermgr.sh \ - $ROOT/test/tests/test_canary.sh \ $ROOT/test/tests/test_env_vars.sh \ $ROOT/test/tests/test_environments/test_python_env.sh \ $ROOT/test/tests/test_fn_update/test_idle_objects_reaper.sh \ @@ -473,14 +474,15 @@ run_all_tests() { $ROOT/test/tests/test_router_cache_invalidation.sh \ $ROOT/test/tests/test_specs/test_spec.sh \ $ROOT/test/tests/test_specs/test_spec_multifile.sh \ - $ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh + $ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh \ + $ROOT/test/tests/test_environments/test_tensorflow_serving_env.sh \ + $ROOT/test/tests/test_environments/test_go_env.sh FAILURES=$? # FIXME: run tests with newdeploy one by one. export JOBS=1 $ROOT/test/run_test.sh \ $ROOT/test/tests/test_backend_newdeploy.sh \ - $ROOT/test/tests/test_environments/test_go_env.sh \ $ROOT/test/tests/test_environments/test_java_builder.sh \ $ROOT/test/tests/test_environments/test_java_env.sh \ $ROOT/test/tests/test_fn_update/test_configmap_update.sh \ diff --git a/test/tests/test_environments/test_tensorflow_serving_env.sh b/test/tests/test_environments/test_tensorflow_serving_env.sh new file mode 100755 index 00000000..e7a020d4 --- /dev/null +++ b/test/tests/test_environments/test_tensorflow_serving_env.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -euo pipefail +source $(dirname $0)/../../utils.sh + +TEST_ID=$(generate_test_id) +echo "TEST_ID = $TEST_ID" + +tmp_dir="/tmp/test-$TEST_ID" +mkdir -p $tmp_dir + +ROOT=$(dirname $0)/../../.. + +cleanup() { + clean_resource_by_id $TEST_ID + rm -rf $tmp_dir +} + +if [ -z "${TEST_NOCLEANUP:-}" ]; then + trap cleanup EXIT +else + log "TEST_NOCLEANUP is set; not cleaning up test artifacts afterwards." +fi + +env=ts-$TEST_ID +fn_poolmgr=hello-ts-poolmgr-$TEST_ID +fn_nd=hello-ts-nd-$TEST_ID + +cd $ROOT/examples/tensorflow-serving + +log "Creating environment for Tensorflow Serving" +fission env create --name $env --image $TS_RUNTIME_IMAGE --version 2 --period 5 + +zip -r half_plus_two.zip ./half_plus_two + +pkgName=$(fission package create --deploy half_plus_two.zip --env $env| cut -f2 -d' '| tr -d \') + +# wait for build to finish at most 90s +timeout 90 bash -c "waitBuild $pkgName" + +log "Creating pool manager & new deployment function for Tensorflow Serving" +fission fn create --name $fn_poolmgr --env $env --pkg $pkgName --entrypoint "half_plus_two" +fission fn create --name $fn_nd --env $env --pkg $pkgName --entrypoint "half_plus_two" --executortype newdeploy + +log "Creating route for new deployment function" +fission route create --function $fn_poolmgr --url /$fn_poolmgr --method POST +fission route create --function $fn_nd --url /$fn_nd --method POST + +log "Waiting for router & pools to catch up" +sleep 5 + +body='{\"instances\": [1.0, 2.0, 5.0]}' +expect='\"predictions\": \[2.5, 3.0, 4.5' + +log "Testing pool manager function" +timeout 60 bash -c "test_post_route $fn_poolmgr \"$body\" \"$expect\"" + +log "Testing new deployment function" +timeout 60 bash -c "test_post_route $fn_nd \"$body\" \"$expect\"" + +log "Test PASSED" diff --git a/test/utils.sh b/test/utils.sh index 5424b40b..a41684e8 100755 --- a/test/utils.sh +++ b/test/utils.sh @@ -71,6 +71,35 @@ test_fn() { } export -f test_fn +test_post_route() { + # Doing an HTTP POST on the function's route + # Checking for valid response + url="http://$FISSION_ROUTER/$1" + body=$2 + expect=$3 + + set +e + while true; do + log "test_post_route: call curl" + resp=$(curl --silent --show-error -d "$body" -X POST "$url") + status_code=$? + if [ $status_code -ne 0 ]; then + log "test_post_route: curl failed ($status_code). Retrying ..." + sleep 1 + continue + fi + if ! (echo $resp | grep "$expect" > /dev/null); then + log "test_post_route: resp = '$resp' expect = '$expect'" + log "test_post_route: expected string not found. Retrying ..." + sleep 1 + continue + fi + break + done + set -e +} +export -f test_post_route + wait_for_builder() { env=$1 JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' @@ -108,7 +137,9 @@ export -f waitBuild export FISSION_NAMESPACE=${FISSION_NAMESPACE:-fission} export FUNCTION_NAMESPACE=${FUNCTION_NAMESPACE:-fission-function} -export FISSION_ROUTER=$(kubectl -n $FISSION_NAMESPACE get svc router -o jsonpath='{...ip}') +router=$(kubectl -n $FISSION_NAMESPACE get svc router -o jsonpath='{...ip}') + +export FISSION_ROUTER=${FISSION_ROUTER:-$router} export FISSION_NATS_STREAMING_URL="http://defaultFissionAuthToken@$(kubectl -n $FISSION_NAMESPACE get svc nats-streaming -o jsonpath='{...ip}:{.spec.ports[0].port}')" ## Parameters used by some specific test cases @@ -119,4 +150,5 @@ export GO_BUILDER_IMAGE=${GO_BUILDER_IMAGE:-fission/go-builder} export JVM_RUNTIME_IMAGE=${JVM_RUNTIME_IMAGE:-fission/jvm-env} export JVM_BUILDER_IMAGE=${JVM_BUILDER_IMAGE:-fission/jvm-builder} export NODE_RUNTIME_IMAGE=${NODE_RUNTIME_IMAGE:-fission/node-env} +export TS_RUNTIME_IMAGE=${TS_RUNTIME_IMAGE:-fission/tensorflow-serving-env}