Fix requests are sent to unready function pod (newdeploy) (#1005)

* Refactor specialization process
* Separate readiness and liveness probe to different routes

For newdeploy, readiness probe should check whether a fetcher specializes env container successfully or not. In this commit, fetcher returns the actual state of current specialization status instead of returning 200ok directly.

environments/fetcher/cmd/main.go

@@ -1,20 +1,15 @@
 package main
 
 import (
-	"bytes"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
-	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
 	"runtime/debug"
-	"strconv"
 	"syscall"
-	"time"
 
 	"github.com/fission/fission"
 	"github.com/fission/fission/environments/fetcher"
@@ -38,8 +33,7 @@ func main() {
 
 	flag.Usage = fetcherUsage
 	specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup")
-	fetchPayload := flag.String("fetch-request", "", "JSON Payload for fetch request")
-	loadPayload := flag.String("load-request", "", "JSON payload for Load request")
+	specializePayload := flag.String("specialize-request", "", "JSON payload for specialize request")
 	secretDir := flag.String("secret-dir", "", "Path to shared secrets directory")
 	configDir := flag.String("cfgmap-dir", "", "Path to shared configmap directory")
 
@@ -59,19 +53,44 @@ func main() {
 		}
 	}
 
-	fetcher, err := fetcher.MakeFetcher(dir, *secretDir, *configDir)
+	f, err := fetcher.MakeFetcher(dir, *secretDir, *configDir)
 	if err != nil {
 		log.Fatalf("Error making fetcher: %v", err)
 	}
 
-	if *specializeOnStart {
-		specializePod(fetcher, fetchPayload, loadPayload)
-	}
+	readyToServe := false
+
+	// do specialization in other goroutine to prevent blocking in newdeploy
+	go func() {
+		if *specializeOnStart {
+			var specializeReq fission.FunctionSpecializeRequest
+
+			err := json.Unmarshal([]byte(*specializePayload), &specializeReq)
+			if err != nil {
+				log.Fatalf("Error decoding specialize request: %v", err)
+			}
+
+			err = f.SpecializePod(specializeReq.FetchReq, specializeReq.LoadReq)
+			if err != nil {
+				log.Fatalf("Error specialing function poadt: %v", err)
+			}
+
+			readyToServe = true
+		}
+	}()
 
 	mux := http.NewServeMux()
-	mux.HandleFunc("/", fetcher.FetchHandler)
-	mux.HandleFunc("/upload", fetcher.UploadHandler)
-	mux.HandleFunc("/version", fetcher.VersionHandler)
+	mux.HandleFunc("/fetch", f.FetchHandler)
+	mux.HandleFunc("/specialize", f.SpecializeHandler)
+	mux.HandleFunc("/upload", f.UploadHandler)
+	mux.HandleFunc("/version", f.VersionHandler)
+	mux.HandleFunc("/readniess-healthz", func(w http.ResponseWriter, r *http.Request) {
+		if !*specializeOnStart || readyToServe {
+			w.WriteHeader(http.StatusOK)
+		} else {
+			w.WriteHeader(http.StatusServiceUnavailable)
+		}
+	})
 	mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	})
@@ -81,75 +100,5 @@ func main() {
 }
 
 func fetcherUsage() {
-	fmt.Printf("Usage: fetcher [-specialize-on-startup] [-fetch-request <json>] [-load-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path> \n")
-}
-
-func specializePod(f *fetcher.Fetcher, fetchPayload *string, loadPayload *string) {
-	// Fetch code
-	var fetchReq fetcher.FetchRequest
-	err := json.Unmarshal([]byte(*fetchPayload), &fetchReq)
-	if err != nil {
-		log.Fatalf("Error parsing fetch request: %v", err)
-	}
-	_, err = f.Fetch(fetchReq)
-	if err != nil {
-		log.Fatalf("Error fetching: %v", err)
-	}
-
-	_, err = f.FetchSecretsAndCfgMaps(fetchReq.Secrets, fetchReq.ConfigMaps)
-	if err != nil {
-		log.Fatalf("Error fetching secrets/configmaps: %v", err)
-		return
-	}
-
-	// Specialize the pod
-
-	envVersion, err := strconv.Atoi(os.Getenv("ENV_VERSION"))
-	if err != nil {
-		log.Fatalf("Error parsing environment version %v, error: %v", os.Getenv("ENV_VERSION"), err)
-	}
-
-	maxRetries := 30
-	var contentType string
-	var specializeURL string
-	var reader *bytes.Reader
-
-	if envVersion >= 2 {
-		contentType = "application/json"
-		specializeURL = "http://localhost:8888/v2/specialize"
-		reader = bytes.NewReader([]byte(*loadPayload))
-	} else {
-		contentType = "text/plain"
-		specializeURL = "http://localhost:8888/specialize"
-		reader = bytes.NewReader([]byte{})
-	}
-
-	for i := 0; i < maxRetries; i++ {
-		resp, err := http.Post(specializeURL, contentType, reader)
-		if err == nil && resp.StatusCode < 300 {
-			// Success
-			resp.Body.Close()
-			break
-		}
-
-		// Only retry for the specific case of a connection error.
-		if urlErr, ok := err.(*url.Error); ok {
-			if netErr, ok := urlErr.Err.(*net.OpError); ok {
-				if netErr.Op == "dial" {
-					if i < maxRetries-1 {
-						time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
-						log.Printf("Error connecting to pod (%v), retrying", netErr)
-						continue
-					}
-				}
-			}
-		}
-
-		if err == nil {
-			err = fission.MakeErrorFromHTTP(resp)
-		}
-		log.Printf("Failed to specialize pod: %v", err)
-		return
-	}
-
+	fmt.Printf("Usage: fetcher [-specialize-on-startup] [-specialize-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path> \n")
 }