Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix requests are sent to unready function pod (newdeploy) (#1005)

Browse Source 
* Refactor specialization process
* Separate readiness and liveness probe to different routes

For newdeploy, readiness probe should check whether a fetcher specializes env container successfully or not. In this commit, fetcher returns the actual state of current specialization status instead of returning 200ok directly.
This commit is contained in:
Ta-Ching Chen
2018-12-27 16:29:31 +08:00
committed by GitHub
parent 04e6433d5d
commit 37846a5827
9 changed files with 312 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files
environments/fetcher/client/client.go
+58 -52
View File
@@ -6,10 +6,10 @@ import (
"io/ioutil"
"log"
"net/http"
"strings"
"time"
"github.com/fission/fission"
"github.com/fission/fission/environments/fetcher"
)
type (
@@ -20,68 +20,74 @@ type (
func MakeClient(fetcherUrl string) *Client {
return &Client{
url: fetcherUrl,
url: strings.TrimSuffix(fetcherUrl, "/"),
}
}
func (c *Client) Fetch(fr *fetcher.FetchRequest) error {
body, err := json.Marshal(fr)
if err != nil {
return err
}
maxRetries := 20
var resp *http.Response
for i := 0; i < maxRetries; i++ {
resp, err = http.Post(c.url, "application/json", bytes.NewReader(body))
if err == nil {
if resp.StatusCode == 200 {
resp.Body.Close()
return nil
}
err = fission.MakeErrorFromHTTP(resp)
}
if i < maxRetries-1 {
time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error fetching package (%v), retrying", err)
continue
}
log.Printf("Failed to fetch: %v", err)
return err
}
return nil
func (c *Client) getSpecializeUrl() string {
return c.url + "/specialize"
}
func (c *Client) Upload(fr *fetcher.UploadRequest) (*fetcher.UploadResponse, error) {
body, err := json.Marshal(fr)
if err != nil {
return nil, err
}
resp, err := http.Post(c.url+"/upload", "application/json", bytes.NewReader(body))
if err != nil {
return nil, err
}
defer resp.Body.Close()
func (c *Client) getFetchUrl() string {
return c.url + "/fetch"
}
if resp.StatusCode != 200 {
return nil, fission.MakeErrorFromHTTP(resp)
}
func (c *Client) getUploadUrl() string {
return c.url + "/upload"
}
rBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
func (c *Client) Specialize(req *fission.FunctionSpecializeRequest) error {
_, err := sendRequest(req, c.getSpecializeUrl())
return err
}
uploadResp := fetcher.UploadResponse{}
err = json.Unmarshal([]byte(rBody), &uploadResp)
func (c *Client) Fetch(fr *fission.FunctionFetchRequest) error {
_, err := sendRequest(fr, c.getFetchUrl())
return err
}
func (c *Client) Upload(fr *fission.ArchiveUploadRequest) (*fission.ArchiveUploadResponse, error) {
body, err := sendRequest(fr, c.getUploadUrl())
uploadResp := fission.ArchiveUploadResponse{}
err = json.Unmarshal(body, &uploadResp)
if err != nil {
return nil, err
}
return &uploadResp, nil
}
func sendRequest(req interface{}, url string) ([]byte, error) {
body, err := json.Marshal(req)
if err != nil {
return nil, err
}
maxRetries := 20
var resp *http.Response
for i := 0; i < maxRetries; i++ {
resp, err = http.Post(url, "application/json", bytes.NewReader(body))
if err == nil {
if resp.StatusCode == 200 {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Error reading response body: %v", err)
}
resp.Body.Close()
return body, err
}
err = fission.MakeErrorFromHTTP(resp)
}
if i < maxRetries-1 {
time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error specializing/fetching/uploading package (%v) with url %v, retrying", err, url)
continue
}
}
return nil, err
}
environments/fetcher/cmd/main.go
+34 -85
View File
@@ -1,20 +1,15 @@
package main
import (
"bytes"
"encoding/json"
"flag"
"fmt"
"log"
"net"
"net/http"
"net/url"
"os"
"os/signal"
"runtime/debug"
"strconv"
"syscall"
"time"
"github.com/fission/fission"
"github.com/fission/fission/environments/fetcher"
@@ -38,8 +33,7 @@ func main() {
flag.Usage = fetcherUsage
specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup")
fetchPayload := flag.String("fetch-request", "", "JSON Payload for fetch request")
loadPayload := flag.String("load-request", "", "JSON payload for Load request")
specializePayload := flag.String("specialize-request", "", "JSON payload for specialize request")
secretDir := flag.String("secret-dir", "", "Path to shared secrets directory")
configDir := flag.String("cfgmap-dir", "", "Path to shared configmap directory")
@@ -59,19 +53,44 @@ func main() {
}
}
fetcher, err := fetcher.MakeFetcher(dir, *secretDir, *configDir)
f, err := fetcher.MakeFetcher(dir, *secretDir, *configDir)
if err != nil {
log.Fatalf("Error making fetcher: %v", err)
}
if *specializeOnStart {
specializePod(fetcher, fetchPayload, loadPayload)
}
readyToServe := false
// do specialization in other goroutine to prevent blocking in newdeploy
go func() {
if *specializeOnStart {
var specializeReq fission.FunctionSpecializeRequest
err := json.Unmarshal([]byte(*specializePayload), &specializeReq)
if err != nil {
log.Fatalf("Error decoding specialize request: %v", err)
}
err = f.SpecializePod(specializeReq.FetchReq, specializeReq.LoadReq)
if err != nil {
log.Fatalf("Error specialing function poadt: %v", err)
}
readyToServe = true
}
}()
mux := http.NewServeMux()
mux.HandleFunc("/", fetcher.FetchHandler)
mux.HandleFunc("/upload", fetcher.UploadHandler)
mux.HandleFunc("/version", fetcher.VersionHandler)
mux.HandleFunc("/fetch", f.FetchHandler)
mux.HandleFunc("/specialize", f.SpecializeHandler)
mux.HandleFunc("/upload", f.UploadHandler)
mux.HandleFunc("/version", f.VersionHandler)
mux.HandleFunc("/readniess-healthz", func(w http.ResponseWriter, r *http.Request) {
if !*specializeOnStart || readyToServe {
w.WriteHeader(http.StatusOK)
} else {
w.WriteHeader(http.StatusServiceUnavailable)
}
})
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
})
@@ -81,75 +100,5 @@ func main() {
}
func fetcherUsage() {
fmt.Printf("Usage: fetcher [-specialize-on-startup] [-fetch-request <json>] [-load-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path> \n")
}
func specializePod(f *fetcher.Fetcher, fetchPayload *string, loadPayload *string) {
// Fetch code
var fetchReq fetcher.FetchRequest
err := json.Unmarshal([]byte(*fetchPayload), &fetchReq)
if err != nil {
log.Fatalf("Error parsing fetch request: %v", err)
}
_, err = f.Fetch(fetchReq)
if err != nil {
log.Fatalf("Error fetching: %v", err)
}
_, err = f.FetchSecretsAndCfgMaps(fetchReq.Secrets, fetchReq.ConfigMaps)
if err != nil {
log.Fatalf("Error fetching secrets/configmaps: %v", err)
return
}
// Specialize the pod
envVersion, err := strconv.Atoi(os.Getenv("ENV_VERSION"))
if err != nil {
log.Fatalf("Error parsing environment version %v, error: %v", os.Getenv("ENV_VERSION"), err)
}
maxRetries := 30
var contentType string
var specializeURL string
var reader *bytes.Reader
if envVersion >= 2 {
contentType = "application/json"
specializeURL = "http://localhost:8888/v2/specialize"
reader = bytes.NewReader([]byte(*loadPayload))
} else {
contentType = "text/plain"
specializeURL = "http://localhost:8888/specialize"
reader = bytes.NewReader([]byte{})
}
for i := 0; i < maxRetries; i++ {
resp, err := http.Post(specializeURL, contentType, reader)
if err == nil && resp.StatusCode < 300 {
// Success
resp.Body.Close()
break
}
// Only retry for the specific case of a connection error.
if urlErr, ok := err.(*url.Error); ok {
if netErr, ok := urlErr.Err.(*net.OpError); ok {
if netErr.Op == "dial" {
if i < maxRetries-1 {
time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error connecting to pod (%v), retrying", netErr)
continue
}
}
}
}
if err == nil {
err = fission.MakeErrorFromHTTP(resp)
}
log.Printf("Failed to specialize pod: %v", err)
return
}
fmt.Printf("Usage: fetcher [-specialize-on-startup] [-specialize-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path> \n")
}
environments/fetcher/fetcher.go
+114 -42
View File
@@ -1,20 +1,23 @@
package fetcher
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"time"
"github.com/mholt/archiver"
"github.com/pkg/errors"
"github.com/satori/go.uuid"
k8serr "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -26,34 +29,6 @@ import (
)
type (
FetchRequestType int
FetchRequest struct {
FetchType FetchRequestType `json:"fetchType"`
Package metav1.ObjectMeta `json:"package"`
Url string `json:"url"`
StorageSvcUrl string `json:"storagesvcurl"`
Filename string `json:"filename"`
Secrets []fission.SecretReference `json:"secretList"`
ConfigMaps []fission.ConfigMapReference `json:"configMapList"`
KeepArchive bool `json:"keeparchive"`
}
// UploadRequest send from builder manager describes which
// deployment package should be upload to storage service.
UploadRequest struct {
Filename string `json:"filename"`
StorageSvcUrl string `json:"storagesvcurl"`
ArchivePackage bool `json:"archivepackage"`
}
// UploadResponse defines the download url of an archive and
// its checksum.
UploadResponse struct {
ArchiveDownloadUrl string `json:"archiveDownloadUrl"`
Checksum fission.Checksum `json:"checksum"`
}
Fetcher struct {
sharedVolumePath string
sharedSecretPath string
@@ -63,12 +38,6 @@ type (
}
)
const (
FETCH_SOURCE = iota
FETCH_DEPLOYMENT
FETCH_URL // remove this?
)
func makeVolumeDir(dirPath string) {
err := os.MkdirAll(dirPath, os.ModeDir|0700)
if err != nil {
@@ -198,7 +167,7 @@ func (fetcher *Fetcher) FetchHandler(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var req FetchRequest
var req fission.FunctionFetchRequest
err = json.Unmarshal(body, &req)
if err != nil {
log.Printf("Error reading request body: %v", err)
@@ -225,9 +194,42 @@ func (fetcher *Fetcher) FetchHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func (fetcher *Fetcher) SpecializeHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, fmt.Sprintf("only POST is supported on this endpoint, %v received", r.Method), http.StatusMethodNotAllowed)
return
}
// parse request
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Printf("Error reading request body")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var req fission.FunctionSpecializeRequest
err = json.Unmarshal(body, &req)
if err != nil {
log.Printf("Error reading request body: %v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
//log.Printf("fetcher received fetch request and started downloading: %v", req)
err = fetcher.SpecializePod(req.FetchReq, req.LoadReq)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
// all done
w.WriteHeader(http.StatusOK)
}
// Fetch takes FetchRequest and makes the fetch call
// It returns the HTTP code and error if any
func (fetcher *Fetcher) Fetch(req FetchRequest) (int, error) {
func (fetcher *Fetcher) Fetch(req fission.FunctionFetchRequest) (int, error) {
// check that the requested filename is not an empty string and error out if so
if len(req.Filename) == 0 {
e := fmt.Sprintf("Fetch request received for an empty file name, request: %v", req)
@@ -244,7 +246,7 @@ func (fetcher *Fetcher) Fetch(req FetchRequest) (int, error) {
tmpFile := req.Filename + ".tmp"
tmpPath := filepath.Join(fetcher.sharedVolumePath, tmpFile)
if req.FetchType == FETCH_URL {
if req.FetchType == fission.FETCH_URL {
// fetch the file and save it to the tmp path
err := downloadUrl(req.Url, tmpPath)
if err != nil {
@@ -262,9 +264,9 @@ func (fetcher *Fetcher) Fetch(req FetchRequest) (int, error) {
}
var archive *fission.Archive
if req.FetchType == FETCH_SOURCE {
if req.FetchType == fission.FETCH_SOURCE {
archive = &pkg.Spec.Source
} else if req.FetchType == FETCH_DEPLOYMENT {
} else if req.FetchType == fission.FETCH_DEPLOYMENT {
// sometimes, the user may invoke the function even before the source code is built into a deploy pkg.
// this results in executor sending a fetch request of type FETCH_DEPLOYMENT and since pkg.Spec.Deployment.Url will be empty,
// we hit this "Get : unsupported protocol scheme "" error.
@@ -418,7 +420,7 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
return
}
var req UploadRequest
var req fission.ArchiveUploadRequest
err = json.Unmarshal(body, &req)
if err != nil {
log.Printf("Error reading request body: %v", err)
@@ -468,7 +470,7 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
return
}
resp := UploadResponse{
resp := fission.ArchiveUploadResponse{
ArchiveDownloadUrl: ssClient.GetUrl(fileID),
Checksum: *sum,
}
@@ -523,3 +525,73 @@ func (fetcher *Fetcher) unarchive(src string, dst string) error {
}
return nil
}
func (fetcher *Fetcher) SpecializePod(fetchReq fission.FunctionFetchRequest, loadReq fission.FunctionLoadRequest) error {
startTime := time.Now()
defer func() {
elapsed := time.Since(startTime)
log.Printf("Elapsed time in fetch request = %v", elapsed)
}()
_, err := fetcher.Fetch(fetchReq)
if err != nil {
return errors.Wrap(err, "Error fetching deploy package")
}
_, err = fetcher.FetchSecretsAndCfgMaps(fetchReq.Secrets, fetchReq.ConfigMaps)
if err != nil {
return errors.Wrap(err, "Error fetching secrets/configmaps")
}
// Specialize the pod
maxRetries := 30
var contentType string
var specializeURL string
var reader *bytes.Reader
loadPayload, err := json.Marshal(loadReq)
if err != nil {
return errors.Wrap(err, "Error encoding load request")
}
if loadReq.EnvVersion >= 2 {
contentType = "application/json"
specializeURL = "http://localhost:8888/v2/specialize"
reader = bytes.NewReader(loadPayload)
} else {
contentType = "text/plain"
specializeURL = "http://localhost:8888/specialize"
reader = bytes.NewReader([]byte{})
}
for i := 0; i < maxRetries; i++ {
resp, err := http.Post(specializeURL, contentType, reader)
if err == nil && resp.StatusCode < 300 {
// Success
resp.Body.Close()
return nil
}
// Only retry for the specific case of a connection error.
if urlErr, ok := err.(*url.Error); ok {
if netErr, ok := urlErr.Err.(*net.OpError); ok {
if netErr.Op == "dial" {
if i < maxRetries-1 {
time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error connecting to pod (%v), retrying", netErr)
continue
}
}
}
}
if err == nil {
err = fission.MakeErrorFromHTTP(resp)
}
return errors.Wrap(err, "Error specializing function pod")
}
return errors.Wrap(err, fmt.Sprintf("Error specializing function pod after %v times", maxRetries))
}