Move packages to proejct/pkg to follow go project folder structure convention (#1190)
This commit is contained in:
Ta-Ching Chen
@@ -1,105 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.opencensus.io/plugin/ochttp"
|
||||
"golang.org/x/net/context/ctxhttp"
|
||||
|
||||
"github.com/fission/fission"
|
||||
)
|
||||
|
||||
type (
|
||||
Client struct {
|
||||
logger *zap.Logger
|
||||
url string
|
||||
httpClient *http.Client
|
||||
}
|
||||
)
|
||||
|
||||
func MakeClient(logger *zap.Logger, fetcherUrl string) *Client {
|
||||
return &Client{
|
||||
logger: logger.Named("fetcher_client"),
|
||||
url: strings.TrimSuffix(fetcherUrl, "/"),
|
||||
httpClient: &http.Client{
|
||||
Transport: &ochttp.Transport{},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) getSpecializeUrl() string {
|
||||
return c.url + "/specialize"
|
||||
}
|
||||
|
||||
func (c *Client) getFetchUrl() string {
|
||||
return c.url + "/fetch"
|
||||
}
|
||||
|
||||
func (c *Client) getUploadUrl() string {
|
||||
return c.url + "/upload"
|
||||
}
|
||||
|
||||
func (c *Client) Specialize(ctx context.Context, req *fission.FunctionSpecializeRequest) error {
|
||||
_, err := sendRequest(c.logger, ctx, c.httpClient, req, c.getSpecializeUrl())
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Client) Fetch(ctx context.Context, fr *fission.FunctionFetchRequest) error {
|
||||
_, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getFetchUrl())
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Client) Upload(ctx context.Context, fr *fission.ArchiveUploadRequest) (*fission.ArchiveUploadResponse, error) {
|
||||
body, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getUploadUrl())
|
||||
|
||||
uploadResp := fission.ArchiveUploadResponse{}
|
||||
err = json.Unmarshal(body, &uploadResp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &uploadResp, nil
|
||||
}
|
||||
|
||||
func sendRequest(logger *zap.Logger, ctx context.Context, httpClient *http.Client, req interface{}, url string) ([]byte, error) {
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
maxRetries := 20
|
||||
var resp *http.Response
|
||||
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
resp, err = ctxhttp.Post(ctx, httpClient, url, "application/json", bytes.NewReader(body))
|
||||
|
||||
if err == nil {
|
||||
if resp.StatusCode == 200 {
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logger.Error("error reading response body", zap.Error(err))
|
||||
}
|
||||
resp.Body.Close()
|
||||
return body, err
|
||||
}
|
||||
err = fission.MakeErrorFromHTTP(resp)
|
||||
}
|
||||
|
||||
if i < maxRetries-1 {
|
||||
time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
|
||||
logger.Error("error specializing/fetching/uploading package, retrying", zap.Error(err), zap.String("url", url))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
FROM golang:1.11-alpine as builder
|
||||
RUN apk add bash ca-certificates git gcc g++ libc-dev
|
||||
|
||||
ARG GITCOMMIT=unknown
|
||||
# E.g. GITCOMMIT=$(git rev-parse HEAD)
|
||||
|
||||
ARG BUILDVERSION=unknown
|
||||
# E.g. BUILDVERSION=$(git rev-parse HEAD)
|
||||
|
||||
ARG BUILDDATE=unknown
|
||||
# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
ARG GOPKG=github.com/fission/fission
|
||||
COPY . /go/src/${GOPKG}
|
||||
WORKDIR /go/src/${GOPKG}/environments/fetcher/cmd
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
|
||||
-o /go/bin/fetcher \
|
||||
-gcflags=-trimpath=$GOPATH \
|
||||
-asmflags=-trimpath=$GOPATH \
|
||||
-ldflags "-X github.com/fission/fission.GitCommit=${GITCOMMIT} -X github.com/fission/fission.BuildDate=${BUILDDATE} -X github.com/fission/fission.Version=${BUILDVERSION}"
|
||||
|
||||
FROM alpine:3.4
|
||||
COPY --from=builder /go/bin/fetcher /
|
||||
EXPOSE 8000
|
||||
|
||||
ENTRYPOINT ["/fetcher"]
|
||||
@@ -1,130 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"go.opencensus.io/exporter/jaeger"
|
||||
"go.opencensus.io/plugin/ochttp"
|
||||
"go.opencensus.io/trace"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/fission/fission"
|
||||
"github.com/fission/fission/environments/fetcher"
|
||||
)
|
||||
|
||||
func registerTraceExporter(collectorEndpoint string) error {
|
||||
if collectorEndpoint == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
serviceName := "Fission-Fetcher"
|
||||
exporter, err := jaeger.NewExporter(jaeger.Options{
|
||||
CollectorEndpoint: collectorEndpoint,
|
||||
Process: jaeger.Process{
|
||||
ServiceName: serviceName,
|
||||
Tags: []jaeger.Tag{
|
||||
jaeger.BoolTag("fission", true),
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
trace.RegisterExporter(exporter)
|
||||
trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
|
||||
return nil
|
||||
}
|
||||
|
||||
// Usage: fetcher <shared volume path>
|
||||
func main() {
|
||||
logger, err := zap.NewProduction()
|
||||
if err != nil {
|
||||
log.Fatalf("can't initialize zap logger: %v", err)
|
||||
}
|
||||
defer logger.Sync()
|
||||
|
||||
flag.Usage = fetcherUsage
|
||||
collectorEndpoint := flag.String("jaeger-collector-endpoint", "", "")
|
||||
specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup")
|
||||
specializePayload := flag.String("specialize-request", "", "JSON payload for specialize request")
|
||||
secretDir := flag.String("secret-dir", "", "Path to shared secrets directory")
|
||||
configDir := flag.String("cfgmap-dir", "", "Path to shared configmap directory")
|
||||
|
||||
flag.Parse()
|
||||
if flag.NArg() == 0 {
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
dir := flag.Arg(0)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll(dir, os.ModeDir|0700)
|
||||
if err != nil {
|
||||
logger.Fatal("error creating directory", zap.Error(err), zap.String("directory", dir))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := registerTraceExporter(*collectorEndpoint); err != nil {
|
||||
logger.Fatal("could not register trace exporter", zap.Error(err), zap.String("collector_endpoint", *collectorEndpoint))
|
||||
}
|
||||
|
||||
f, err := fetcher.MakeFetcher(logger, dir, *secretDir, *configDir)
|
||||
if err != nil {
|
||||
logger.Fatal("error making fetcher", zap.Error(err))
|
||||
}
|
||||
|
||||
readyToServe := false
|
||||
|
||||
// do specialization in other goroutine to prevent blocking in newdeploy
|
||||
go func() {
|
||||
if *specializeOnStart {
|
||||
var specializeReq fission.FunctionSpecializeRequest
|
||||
|
||||
err := json.Unmarshal([]byte(*specializePayload), &specializeReq)
|
||||
if err != nil {
|
||||
logger.Fatal("error decoding specialize request", zap.Error(err))
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err = f.SpecializePod(ctx, specializeReq.FetchReq, specializeReq.LoadReq)
|
||||
if err != nil {
|
||||
logger.Fatal("error specializing function pod", zap.Error(err))
|
||||
}
|
||||
|
||||
readyToServe = true
|
||||
}
|
||||
}()
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/fetch", f.FetchHandler)
|
||||
mux.HandleFunc("/specialize", f.SpecializeHandler)
|
||||
mux.HandleFunc("/upload", f.UploadHandler)
|
||||
mux.HandleFunc("/version", f.VersionHandler)
|
||||
mux.HandleFunc("/readniess-healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !*specializeOnStart || readyToServe {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
} else {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
}
|
||||
})
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})
|
||||
|
||||
logger.Info("fetcher ready to receive requests")
|
||||
http.ListenAndServe(":8000", &ochttp.Handler{
|
||||
Handler: mux,
|
||||
})
|
||||
}
|
||||
|
||||
func fetcherUsage() {
|
||||
fmt.Println("Usage: fetcher [-specialize-on-startup] [-specialize-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path>")
|
||||
}
|
||||
@@ -1,303 +0,0 @@
|
||||
package container
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
"github.com/fission/fission"
|
||||
crd "github.com/fission/fission/crd"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
fetcherImage string
|
||||
fetcherImagePullPolicy apiv1.PullPolicy
|
||||
|
||||
resourceRequirements apiv1.ResourceRequirements
|
||||
|
||||
// used by generic pool when creating env deployment to specify the share volume path for fetcher & env
|
||||
// change this may break v1 compatibility, since most of the v1 environments have hard-coded "/userfunc" in loading path
|
||||
sharedMountPath string
|
||||
sharedSecretPath string
|
||||
sharedCfgMapPath string
|
||||
|
||||
dockerRegistryAuthDomain string
|
||||
dockerRegistryUsername string
|
||||
dockerRegistryPassword string
|
||||
|
||||
serviceAccount string
|
||||
|
||||
jaegerCollectorEndpoint string
|
||||
}
|
||||
|
||||
func getFetcherResources() (apiv1.ResourceRequirements, error) {
|
||||
mincpu, err := resource.ParseQuantity(os.Getenv("FETCHER_MINCPU"))
|
||||
if err != nil {
|
||||
return apiv1.ResourceRequirements{}, err
|
||||
}
|
||||
|
||||
minmem, err := resource.ParseQuantity(os.Getenv("FETCHER_MINMEM"))
|
||||
if err != nil {
|
||||
return apiv1.ResourceRequirements{}, err
|
||||
}
|
||||
|
||||
maxcpu, err := resource.ParseQuantity(os.Getenv("FETCHER_MAXCPU"))
|
||||
if err != nil {
|
||||
return apiv1.ResourceRequirements{}, err
|
||||
}
|
||||
|
||||
maxmem, err := resource.ParseQuantity(os.Getenv("FETCHER_MAXMEM"))
|
||||
if err != nil {
|
||||
return apiv1.ResourceRequirements{}, err
|
||||
}
|
||||
|
||||
return apiv1.ResourceRequirements{
|
||||
Requests: map[apiv1.ResourceName]resource.Quantity{
|
||||
apiv1.ResourceCPU: mincpu,
|
||||
apiv1.ResourceMemory: minmem,
|
||||
},
|
||||
Limits: map[apiv1.ResourceName]resource.Quantity{
|
||||
apiv1.ResourceCPU: maxcpu,
|
||||
apiv1.ResourceMemory: maxmem,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func MakeFetcherConfig(sharedMountPath string) (*Config, error) {
|
||||
resources, err := getFetcherResources()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fetcherImage := os.Getenv("FETCHER_IMAGE")
|
||||
if len(fetcherImage) == 0 {
|
||||
fetcherImage = "fission/fetcher"
|
||||
}
|
||||
|
||||
fetcherImagePullPolicy := os.Getenv("FETCHER_IMAGE_PULL_POLICY")
|
||||
if len(fetcherImagePullPolicy) == 0 {
|
||||
fetcherImagePullPolicy = "IfNotPresent"
|
||||
}
|
||||
|
||||
return &Config{
|
||||
resourceRequirements: resources,
|
||||
fetcherImage: fetcherImage,
|
||||
fetcherImagePullPolicy: fission.GetImagePullPolicy(fetcherImagePullPolicy),
|
||||
sharedMountPath: sharedMountPath,
|
||||
sharedSecretPath: "/secrets",
|
||||
sharedCfgMapPath: "/configs",
|
||||
jaegerCollectorEndpoint: os.Getenv("OPENCENSUS_TRACE_JAEGER_COLLECTOR_ENDPOINT"),
|
||||
serviceAccount: fission.FissionFetcherSA,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (cfg *Config) SetupServiceAccount(kubernetesClient *kubernetes.Clientset, namespace string, context interface{}) error {
|
||||
_, err := fission.SetupSA(kubernetesClient, fission.FissionFetcherSA, namespace)
|
||||
if err != nil {
|
||||
log.Printf("Error : %v creating %s in ns : %s for: %#v", err, fission.FissionFetcherSA, namespace, context)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cfg *Config) SharedMountPath() string {
|
||||
return cfg.sharedMountPath
|
||||
}
|
||||
|
||||
func (cfg *Config) NewSpecializeRequest(fn *crd.Function, env *crd.Environment) fission.FunctionSpecializeRequest {
|
||||
// for backward compatibility, since most v1 env
|
||||
// still try to load user function from hard coded
|
||||
// path /userfunc/user
|
||||
targetFilename := "user"
|
||||
if env.Spec.Version >= 2 {
|
||||
targetFilename = string(fn.Metadata.UID)
|
||||
}
|
||||
|
||||
return fission.FunctionSpecializeRequest{
|
||||
FetchReq: fission.FunctionFetchRequest{
|
||||
FetchType: fission.FETCH_DEPLOYMENT,
|
||||
Package: metav1.ObjectMeta{
|
||||
Namespace: fn.Spec.Package.PackageRef.Namespace,
|
||||
Name: fn.Spec.Package.PackageRef.Name,
|
||||
},
|
||||
Filename: targetFilename,
|
||||
Secrets: fn.Spec.Secrets,
|
||||
ConfigMaps: fn.Spec.ConfigMaps,
|
||||
KeepArchive: env.Spec.KeepArchive,
|
||||
},
|
||||
LoadReq: fission.FunctionLoadRequest{
|
||||
FilePath: filepath.Join(cfg.sharedMountPath, targetFilename),
|
||||
FunctionName: fn.Spec.Package.FunctionName,
|
||||
FunctionMetadata: &fn.Metadata,
|
||||
EnvVersion: env.Spec.Version,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg *Config) AddFetcherToPodSpec(podSpec *apiv1.PodSpec, mainContainerName string) error {
|
||||
return cfg.addFetcherToPodSpecWithCommand(podSpec, mainContainerName, cfg.fetcherCommand())
|
||||
}
|
||||
|
||||
func (cfg *Config) AddSpecializingFetcherToPodSpec(podSpec *apiv1.PodSpec, mainContainerName string, fn *crd.Function, env *crd.Environment) error {
|
||||
specializeReq := cfg.NewSpecializeRequest(fn, env)
|
||||
specializePayload, err := json.Marshal(specializeReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return cfg.addFetcherToPodSpecWithCommand(
|
||||
podSpec,
|
||||
mainContainerName,
|
||||
cfg.fetcherCommand(
|
||||
"-specialize-on-startup",
|
||||
"-specialize-request", string(specializePayload),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func (cfg *Config) fetcherCommand(extraArgs ...string) []string {
|
||||
command := []string{"/fetcher",
|
||||
"-secret-dir", cfg.sharedSecretPath,
|
||||
"-cfgmap-dir", cfg.sharedCfgMapPath,
|
||||
"-jaeger-collector-endpoint", cfg.jaegerCollectorEndpoint,
|
||||
}
|
||||
|
||||
command = append(command, extraArgs...)
|
||||
command = append(command, cfg.sharedMountPath)
|
||||
return command
|
||||
}
|
||||
|
||||
func (cfg *Config) volumesWithMounts() ([]apiv1.Volume, []apiv1.VolumeMount) {
|
||||
volumes := []apiv1.Volume{
|
||||
{
|
||||
Name: fission.SharedVolumeUserfunc,
|
||||
VolumeSource: apiv1.VolumeSource{
|
||||
EmptyDir: &apiv1.EmptyDirVolumeSource{},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: fission.SharedVolumeSecrets,
|
||||
VolumeSource: apiv1.VolumeSource{
|
||||
EmptyDir: &apiv1.EmptyDirVolumeSource{},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: fission.SharedVolumeConfigmaps,
|
||||
VolumeSource: apiv1.VolumeSource{
|
||||
EmptyDir: &apiv1.EmptyDirVolumeSource{},
|
||||
},
|
||||
},
|
||||
}
|
||||
mounts := []apiv1.VolumeMount{
|
||||
{
|
||||
Name: fission.SharedVolumeUserfunc,
|
||||
MountPath: cfg.sharedMountPath,
|
||||
},
|
||||
{
|
||||
Name: fission.SharedVolumeSecrets,
|
||||
MountPath: cfg.sharedSecretPath,
|
||||
},
|
||||
{
|
||||
Name: fission.SharedVolumeConfigmaps,
|
||||
MountPath: cfg.sharedCfgMapPath,
|
||||
},
|
||||
}
|
||||
|
||||
return volumes, mounts
|
||||
}
|
||||
|
||||
func (cfg *Config) addFetcherToPodSpecWithCommand(podSpec *apiv1.PodSpec, mainContainerName string, command []string) error {
|
||||
volumes, mounts := cfg.volumesWithMounts()
|
||||
c := apiv1.Container{
|
||||
Name: "fetcher",
|
||||
Command: command,
|
||||
Image: cfg.fetcherImage,
|
||||
ImagePullPolicy: cfg.fetcherImagePullPolicy,
|
||||
TerminationMessagePath: "/dev/termination-log",
|
||||
VolumeMounts: mounts,
|
||||
Resources: cfg.resourceRequirements,
|
||||
ReadinessProbe: &apiv1.Probe{
|
||||
InitialDelaySeconds: 1,
|
||||
PeriodSeconds: 1,
|
||||
FailureThreshold: 30,
|
||||
Handler: apiv1.Handler{
|
||||
HTTPGet: &apiv1.HTTPGetAction{
|
||||
Path: "/readniess-healthz",
|
||||
Port: intstr.IntOrString{
|
||||
Type: intstr.Int,
|
||||
IntVal: 8000,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
LivenessProbe: &apiv1.Probe{
|
||||
InitialDelaySeconds: 1,
|
||||
PeriodSeconds: 5,
|
||||
Handler: apiv1.Handler{
|
||||
HTTPGet: &apiv1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.IntOrString{
|
||||
Type: intstr.Int,
|
||||
IntVal: 8000,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Pod is removed from endpoints list for service when it's
|
||||
// state became "Termination". We used preStop hook as the
|
||||
// workaround for connection draining since pod maybe shutdown
|
||||
// before grace period expires.
|
||||
// https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods
|
||||
// https://github.com/kubernetes/kubernetes/issues/47576#issuecomment-308900172
|
||||
if podSpec.TerminationGracePeriodSeconds != nil {
|
||||
c.Lifecycle = &apiv1.Lifecycle{
|
||||
PreStop: &apiv1.Handler{
|
||||
Exec: &apiv1.ExecAction{
|
||||
Command: []string{
|
||||
"/bin/sleep",
|
||||
fmt.Sprintf("%v", *podSpec.TerminationGracePeriodSeconds),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
found := false
|
||||
for ix, container := range podSpec.Containers {
|
||||
if container.Name != mainContainerName {
|
||||
continue
|
||||
}
|
||||
|
||||
found = true
|
||||
container.VolumeMounts = append(container.VolumeMounts, mounts...)
|
||||
podSpec.Containers[ix] = container
|
||||
}
|
||||
if !found {
|
||||
existingContainerNames := make([]string, len(podSpec.Containers))
|
||||
for _, existingContainer := range podSpec.Containers {
|
||||
existingContainerNames = append(existingContainerNames, existingContainer.Name)
|
||||
}
|
||||
return fmt.Errorf("Could not find main container '%s' in given PodSpec. Found: %v",
|
||||
mainContainerName,
|
||||
existingContainerNames)
|
||||
}
|
||||
|
||||
podSpec.Volumes = append(podSpec.Volumes, volumes...)
|
||||
podSpec.Containers = append(podSpec.Containers, c)
|
||||
if podSpec.ServiceAccountName == "" {
|
||||
podSpec.ServiceAccountName = fission.FissionFetcherSA
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1,656 +0,0 @@
|
||||
package fetcher
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/mholt/archiver"
|
||||
"github.com/pkg/errors"
|
||||
uuid "github.com/satori/go.uuid"
|
||||
"go.opencensus.io/plugin/ochttp"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/net/context/ctxhttp"
|
||||
k8serr "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
"github.com/fission/fission"
|
||||
"github.com/fission/fission/crd"
|
||||
storageSvcClient "github.com/fission/fission/storagesvc/client"
|
||||
)
|
||||
|
||||
type (
|
||||
Fetcher struct {
|
||||
logger *zap.Logger
|
||||
sharedVolumePath string
|
||||
sharedSecretPath string
|
||||
sharedConfigPath string
|
||||
fissionClient *crd.FissionClient
|
||||
kubeClient *kubernetes.Clientset
|
||||
httpClient *http.Client
|
||||
}
|
||||
)
|
||||
|
||||
func makeVolumeDir(dirPath string) error {
|
||||
return os.MkdirAll(dirPath, os.ModeDir|0700)
|
||||
}
|
||||
|
||||
func MakeFetcher(logger *zap.Logger, sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) {
|
||||
fLogger := logger.Named("fetcher")
|
||||
err := makeVolumeDir(sharedVolumePath)
|
||||
if err != nil {
|
||||
fLogger.Fatal("error creating shared volume directory", zap.Error(err), zap.String("directory", sharedVolumePath))
|
||||
}
|
||||
err = makeVolumeDir(sharedSecretPath)
|
||||
if err != nil {
|
||||
fLogger.Fatal("error creating shared secret directory", zap.Error(err), zap.String("directory", sharedSecretPath))
|
||||
}
|
||||
err = makeVolumeDir(sharedConfigPath)
|
||||
if err != nil {
|
||||
fLogger.Fatal("error creating shared config directory", zap.Error(err), zap.String("directory", sharedConfigPath))
|
||||
}
|
||||
|
||||
fissionClient, kubeClient, _, err := crd.MakeFissionClient()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error making the fission / kube client")
|
||||
}
|
||||
return &Fetcher{
|
||||
logger: fLogger,
|
||||
sharedVolumePath: sharedVolumePath,
|
||||
sharedSecretPath: sharedSecretPath,
|
||||
sharedConfigPath: sharedConfigPath,
|
||||
fissionClient: fissionClient,
|
||||
kubeClient: kubeClient,
|
||||
httpClient: &http.Client{
|
||||
Transport: &ochttp.Transport{},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func downloadUrl(ctx context.Context, httpClient *http.Client, url string, localPath string) error {
|
||||
resp, err := ctxhttp.Get(ctx, httpClient, url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
w, err := os.Create(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer w.Close()
|
||||
|
||||
_, err = io.Copy(w, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// flushing write buffer to file
|
||||
err = w.Sync()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.Chmod(localPath, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getChecksum(path string) (*fission.Checksum, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
hasher := sha256.New()
|
||||
_, err = io.Copy(hasher, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
return &fission.Checksum{
|
||||
Type: fission.ChecksumTypeSHA256,
|
||||
Sum: c,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func verifyChecksum(fileChecksum, checksum *fission.Checksum) error {
|
||||
if checksum.Type != fission.ChecksumTypeSHA256 {
|
||||
return fission.MakeError(fission.ErrorInvalidArgument, "Unsupported checksum type")
|
||||
}
|
||||
if fileChecksum.Sum != checksum.Sum {
|
||||
return fission.MakeError(fission.ErrorChecksumFail, "Checksum validation failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeSecretOrConfigMap(dataMap map[string][]byte, dirPath string) error {
|
||||
for key, val := range dataMap {
|
||||
writeFilePath := filepath.Join(dirPath, key)
|
||||
err := ioutil.WriteFile(writeFilePath, val, 0600)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "Failed to write file %s", writeFilePath)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fetcher *Fetcher) VersionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
fmt.Fprintf(w, fission.BuildInfo().String())
|
||||
}
|
||||
|
||||
func (fetcher *Fetcher) FetchHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, "only POST is supported on this endpoint", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
defer func() {
|
||||
elapsed := time.Since(startTime)
|
||||
fetcher.logger.Info("fetch request done", zap.Duration("elapsed_time", elapsed))
|
||||
}()
|
||||
|
||||
// parse request
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error reading request body", zap.Error(err))
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
var req fission.FunctionFetchRequest
|
||||
err = json.Unmarshal(body, &req)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error parsing request body", zap.Error(err))
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
code, err := fetcher.Fetch(r.Context(), req)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error fetching", zap.Error(err))
|
||||
http.Error(w, err.Error(), code)
|
||||
return
|
||||
}
|
||||
|
||||
fetcher.logger.Info("checking secrets/cfgmaps")
|
||||
code, err = fetcher.FetchSecretsAndCfgMaps(req.Secrets, req.ConfigMaps)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error fetching secrets and config maps", zap.Error(err))
|
||||
http.Error(w, err.Error(), code)
|
||||
return
|
||||
}
|
||||
|
||||
fetcher.logger.Info("completed fetch request")
|
||||
// all done
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (fetcher *Fetcher) SpecializeHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, fmt.Sprintf("only POST is supported on this endpoint, %v received", r.Method), http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
// parse request
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error reading request body", zap.Error(err))
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
var req fission.FunctionSpecializeRequest
|
||||
err = json.Unmarshal(body, &req)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error parsing request body", zap.Error(err))
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
err = fetcher.SpecializePod(r.Context(), req.FetchReq, req.LoadReq)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error specialing pod", zap.Error(err))
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// all done
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
// Fetch takes FetchRequest and makes the fetch call
|
||||
// It returns the HTTP code and error if any
|
||||
func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequest) (int, error) {
|
||||
// check that the requested filename is not an empty string and error out if so
|
||||
if len(req.Filename) == 0 {
|
||||
e := "fetch request received for an empty file name"
|
||||
fetcher.logger.Error(e, zap.Any("request", req))
|
||||
return http.StatusBadRequest, errors.New(fmt.Sprintf("%s, request: %v", e, req))
|
||||
}
|
||||
|
||||
// verify first if the file already exists.
|
||||
if _, err := os.Stat(filepath.Join(fetcher.sharedVolumePath, req.Filename)); err == nil {
|
||||
fetcher.logger.Info("requested file already exists at shared volume - skipping fetch",
|
||||
zap.String("requested_file", req.Filename),
|
||||
zap.String("shared_volume_path", fetcher.sharedVolumePath))
|
||||
return http.StatusOK, nil
|
||||
}
|
||||
|
||||
tmpFile := req.Filename + ".tmp"
|
||||
tmpPath := filepath.Join(fetcher.sharedVolumePath, tmpFile)
|
||||
|
||||
if req.FetchType == fission.FETCH_URL {
|
||||
// fetch the file and save it to the tmp path
|
||||
err := downloadUrl(ctx, fetcher.httpClient, req.Url, tmpPath)
|
||||
if err != nil {
|
||||
e := "failed to download url"
|
||||
fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url))
|
||||
return http.StatusBadRequest, errors.Wrapf(err, "%s: %s", e, req.Url)
|
||||
}
|
||||
} else {
|
||||
// get pkg
|
||||
pkg, err := fetcher.fissionClient.Packages(req.Package.Namespace).Get(req.Package.Name)
|
||||
if err != nil {
|
||||
e := "failed to get package"
|
||||
fetcher.logger.Error(e,
|
||||
zap.String("package_name", req.Package.Name),
|
||||
zap.String("package_namespace", req.Package.Namespace))
|
||||
return http.StatusInternalServerError, errors.Wrap(err, e)
|
||||
}
|
||||
|
||||
var archive *fission.Archive
|
||||
if req.FetchType == fission.FETCH_SOURCE {
|
||||
archive = &pkg.Spec.Source
|
||||
} else if req.FetchType == fission.FETCH_DEPLOYMENT {
|
||||
// sometimes, the user may invoke the function even before the source code is built into a deploy pkg.
|
||||
// this results in executor sending a fetch request of type FETCH_DEPLOYMENT and since pkg.Spec.Deployment.Url will be empty,
|
||||
// we hit this "Get : unsupported protocol scheme "" error.
|
||||
// it may be useful to the user if we can send a more meaningful error in such a scenario.
|
||||
if pkg.Status.BuildStatus != fission.BuildStatusSucceeded && pkg.Status.BuildStatus != fission.BuildStatusNone {
|
||||
e := fmt.Sprintf("cannot fetch deployment: package build status was not %q", fission.BuildStatusSucceeded)
|
||||
fetcher.logger.Error(e,
|
||||
zap.String("package_name", pkg.Metadata.Name),
|
||||
zap.String("package_namespace", pkg.Metadata.Namespace),
|
||||
zap.Any("package_build_status", pkg.Status.BuildStatus))
|
||||
return http.StatusInternalServerError, errors.New(fmt.Sprintf("%s: pkg %s.%s has a status of %s", e, pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Status.BuildStatus))
|
||||
}
|
||||
archive = &pkg.Spec.Deployment
|
||||
}
|
||||
// get package data as literal or by url
|
||||
if len(archive.Literal) > 0 {
|
||||
// write pkg.Literal into tmpPath
|
||||
err = ioutil.WriteFile(tmpPath, archive.Literal, 0600)
|
||||
if err != nil {
|
||||
e := "failed to write file"
|
||||
fetcher.logger.Error(e, zap.Error(err), zap.String("location", tmpPath))
|
||||
return http.StatusInternalServerError, errors.Wrapf(err, "%s %s", e, tmpPath)
|
||||
}
|
||||
} else {
|
||||
// download and verify
|
||||
err := downloadUrl(ctx, fetcher.httpClient, archive.URL, tmpPath)
|
||||
if err != nil {
|
||||
e := "failed to download url"
|
||||
fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url))
|
||||
return http.StatusBadRequest, errors.Wrapf(err, "%s %s", e, req.Url)
|
||||
}
|
||||
|
||||
checksum, err := getChecksum(tmpPath)
|
||||
if err != nil {
|
||||
e := "failed to get checksum"
|
||||
fetcher.logger.Error(e, zap.Error(err))
|
||||
return http.StatusBadRequest, errors.Wrap(err, e)
|
||||
}
|
||||
|
||||
err = verifyChecksum(checksum, &archive.Checksum)
|
||||
if err != nil {
|
||||
e := "failed to verify checksum"
|
||||
fetcher.logger.Error(e, zap.Error(err))
|
||||
return http.StatusBadRequest, errors.Wrap(err, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if archiver.Zip.Match(tmpPath) && !req.KeepArchive {
|
||||
// unarchive tmp file to a tmp unarchive path
|
||||
tmpUnarchivePath := filepath.Join(fetcher.sharedVolumePath, uuid.NewV4().String())
|
||||
err := fetcher.unarchive(tmpPath, tmpUnarchivePath)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error unarchiving",
|
||||
zap.Error(err),
|
||||
zap.String("archive_location", tmpPath),
|
||||
zap.String("target_location", tmpUnarchivePath))
|
||||
return http.StatusInternalServerError, err
|
||||
}
|
||||
|
||||
tmpPath = tmpUnarchivePath
|
||||
}
|
||||
|
||||
// move tmp file to requested filename
|
||||
renamePath := filepath.Join(fetcher.sharedVolumePath, req.Filename)
|
||||
err := fetcher.rename(tmpPath, renamePath)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error renaming file",
|
||||
zap.Error(err),
|
||||
zap.String("original_path", tmpPath),
|
||||
zap.String("rename_path", renamePath))
|
||||
return http.StatusInternalServerError, err
|
||||
}
|
||||
|
||||
fetcher.logger.Info("successfully placed", zap.String("location", renamePath))
|
||||
return http.StatusOK, nil
|
||||
}
|
||||
|
||||
// FetchSecretsAndCfgMaps fetches secrets and configmaps specified by user
|
||||
// It returns the HTTP code and error if any
|
||||
func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference, cfgmaps []fission.ConfigMapReference) (int, error) {
|
||||
if len(secrets) > 0 {
|
||||
for _, secret := range secrets {
|
||||
data, err := fetcher.kubeClient.CoreV1().Secrets(secret.Namespace).Get(secret.Name, metav1.GetOptions{})
|
||||
|
||||
if err != nil {
|
||||
e := "error getting secret from kubeapi"
|
||||
|
||||
httpCode := http.StatusInternalServerError
|
||||
if k8serr.IsNotFound(err) {
|
||||
httpCode = http.StatusNotFound
|
||||
e = "secret was not found in kubeapi"
|
||||
}
|
||||
fetcher.logger.Error(e,
|
||||
zap.Error(err),
|
||||
zap.String("secret_name", secret.Name),
|
||||
zap.String("secret_namespace", secret.Namespace))
|
||||
|
||||
return httpCode, errors.New(e)
|
||||
}
|
||||
|
||||
secretPath := filepath.Join(secret.Namespace, secret.Name)
|
||||
secretDir := filepath.Join(fetcher.sharedSecretPath, secretPath)
|
||||
err = os.MkdirAll(secretDir, os.ModeDir|0644)
|
||||
if err != nil {
|
||||
e := "failed to create directory for secret"
|
||||
fetcher.logger.Error(e,
|
||||
zap.Error(err),
|
||||
zap.String("directory", secretDir),
|
||||
zap.String("secret_name", secret.Name),
|
||||
zap.String("secret_namespace", secret.Namespace))
|
||||
return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, secretDir)
|
||||
}
|
||||
err = writeSecretOrConfigMap(data.Data, secretDir)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("failed to write secret to file location",
|
||||
zap.Error(err),
|
||||
zap.String("location", secretDir),
|
||||
zap.String("secret_name", secret.Name),
|
||||
zap.String("secret_namespace", secret.Namespace))
|
||||
return http.StatusInternalServerError, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(cfgmaps) > 0 {
|
||||
for _, config := range cfgmaps {
|
||||
data, err := fetcher.kubeClient.CoreV1().ConfigMaps(config.Namespace).Get(config.Name, metav1.GetOptions{})
|
||||
|
||||
if err != nil {
|
||||
e := "error getting configmap from kubeapi"
|
||||
|
||||
httpCode := http.StatusInternalServerError
|
||||
if k8serr.IsNotFound(err) {
|
||||
httpCode = http.StatusNotFound
|
||||
e = "configmap was not found in kubeapi"
|
||||
}
|
||||
fetcher.logger.Error(e,
|
||||
zap.Error(err),
|
||||
zap.String("config_map_name", config.Name),
|
||||
zap.String("config_map_namespace", config.Namespace))
|
||||
|
||||
return httpCode, errors.New(e)
|
||||
}
|
||||
|
||||
configPath := filepath.Join(config.Namespace, config.Name)
|
||||
configDir := filepath.Join(fetcher.sharedConfigPath, configPath)
|
||||
err = os.MkdirAll(configDir, os.ModeDir|0644)
|
||||
if err != nil {
|
||||
e := "failed to create directory for configmap"
|
||||
fetcher.logger.Error(e,
|
||||
zap.Error(err),
|
||||
zap.String("directory", configDir),
|
||||
zap.String("config_map_name", config.Name),
|
||||
zap.String("config_map_namespace", config.Namespace))
|
||||
return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, configDir)
|
||||
}
|
||||
configMap := make(map[string][]byte)
|
||||
for key, val := range data.Data {
|
||||
configMap[key] = []byte(val)
|
||||
}
|
||||
err = writeSecretOrConfigMap(configMap, configDir)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("failed to write configmap to file location",
|
||||
zap.Error(err),
|
||||
zap.String("location", configDir),
|
||||
zap.String("config_map_name", config.Name),
|
||||
zap.String("config_map_namespace", config.Namespace))
|
||||
return http.StatusInternalServerError, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return http.StatusOK, nil
|
||||
}
|
||||
|
||||
func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != "POST" {
|
||||
http.Error(w, "only POST is supported on this endpoint", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
defer func() {
|
||||
elapsed := time.Since(startTime)
|
||||
fetcher.logger.Info("upload request done", zap.Duration("elapsed_time", elapsed))
|
||||
}()
|
||||
|
||||
// parse request
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error reading request body", zap.Error(err))
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var req fission.ArchiveUploadRequest
|
||||
err = json.Unmarshal(body, &req)
|
||||
if err != nil {
|
||||
fetcher.logger.Error("error parsing request body", zap.Error(err))
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
fetcher.logger.Info("fetcher received upload request", zap.Any("request", req))
|
||||
|
||||
zipFilename := req.Filename + ".zip"
|
||||
srcFilepath := filepath.Join(fetcher.sharedVolumePath, req.Filename)
|
||||
dstFilepath := filepath.Join(fetcher.sharedVolumePath, zipFilename)
|
||||
|
||||
if req.ArchivePackage {
|
||||
err = fetcher.archive(srcFilepath, dstFilepath)
|
||||
if err != nil {
|
||||
e := "error archiving zip file"
|
||||
fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath))
|
||||
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
err = os.Rename(srcFilepath, dstFilepath)
|
||||
if err != nil {
|
||||
e := "error renaming the archive"
|
||||
fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath))
|
||||
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fetcher.logger.Info("starting upload...")
|
||||
ssClient := storageSvcClient.MakeClient(req.StorageSvcUrl)
|
||||
|
||||
fileID, err := ssClient.Upload(r.Context(), dstFilepath, nil)
|
||||
if err != nil {
|
||||
e := "error uploading zip file"
|
||||
fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath))
|
||||
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
sum, err := getChecksum(dstFilepath)
|
||||
if err != nil {
|
||||
e := "error calculating checksum of zip file"
|
||||
fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath))
|
||||
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
resp := fission.ArchiveUploadResponse{
|
||||
ArchiveDownloadUrl: ssClient.GetUrl(fileID),
|
||||
Checksum: *sum,
|
||||
}
|
||||
|
||||
rBody, err := json.Marshal(resp)
|
||||
if err != nil {
|
||||
e := "error encoding upload response"
|
||||
fetcher.logger.Error(e, zap.Error(err))
|
||||
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
fetcher.logger.Info("completed upload request")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(rBody)
|
||||
}
|
||||
|
||||
func (fetcher *Fetcher) rename(src string, dst string) error {
|
||||
err := os.Rename(src, dst)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to move file")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// archive zips the contents of directory at src into a new zip file
|
||||
// at dst (note that the contents are zipped, not the directory itself).
|
||||
func (fetcher *Fetcher) archive(src string, dst string) error {
|
||||
var files []string
|
||||
target, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to zip file")
|
||||
}
|
||||
if target.IsDir() {
|
||||
// list all
|
||||
fs, _ := ioutil.ReadDir(src)
|
||||
for _, f := range fs {
|
||||
files = append(files, filepath.Join(src, f.Name()))
|
||||
}
|
||||
} else {
|
||||
files = append(files, src)
|
||||
}
|
||||
return archiver.Zip.Make(dst, files)
|
||||
}
|
||||
|
||||
// unarchive is a function that unzips a zip file to destination
|
||||
func (fetcher *Fetcher) unarchive(src string, dst string) error {
|
||||
err := archiver.Zip.Open(src, dst)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to unzip file")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.FunctionFetchRequest, loadReq fission.FunctionLoadRequest) error {
|
||||
startTime := time.Now()
|
||||
defer func() {
|
||||
elapsed := time.Since(startTime)
|
||||
fetcher.logger.Info("specialize request done", zap.Duration("elapsed_time", elapsed))
|
||||
}()
|
||||
|
||||
_, err := fetcher.Fetch(ctx, fetchReq)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error fetching deploy package")
|
||||
}
|
||||
|
||||
_, err = fetcher.FetchSecretsAndCfgMaps(fetchReq.Secrets, fetchReq.ConfigMaps)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error fetching secrets/configs")
|
||||
}
|
||||
|
||||
// Specialize the pod
|
||||
|
||||
maxRetries := 30
|
||||
var contentType string
|
||||
var specializeURL string
|
||||
var reader *bytes.Reader
|
||||
|
||||
loadPayload, err := json.Marshal(loadReq)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error encoding load request")
|
||||
}
|
||||
|
||||
if loadReq.EnvVersion >= 2 {
|
||||
contentType = "application/json"
|
||||
specializeURL = "http://localhost:8888/v2/specialize"
|
||||
reader = bytes.NewReader(loadPayload)
|
||||
fetcher.logger.Info("calling environment v2 specialization endpoint")
|
||||
} else {
|
||||
contentType = "text/plain"
|
||||
specializeURL = "http://localhost:8888/specialize"
|
||||
reader = bytes.NewReader([]byte{})
|
||||
fetcher.logger.Info("calling environment v1 specialization endpoint")
|
||||
}
|
||||
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
resp, err := http.Post(specializeURL, contentType, reader)
|
||||
if err == nil && resp.StatusCode < 300 {
|
||||
// Success
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Only retry for the specific case of a connection error.
|
||||
if urlErr, ok := err.(*url.Error); ok {
|
||||
if netErr, ok := urlErr.Err.(*net.OpError); ok {
|
||||
if netErr.Op == "dial" {
|
||||
if i < maxRetries-1 {
|
||||
time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
|
||||
fetcher.logger.Error("error connecting to function environment pod for specialization request, retrying", zap.Error(netErr))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
err = fission.MakeErrorFromHTTP(resp)
|
||||
}
|
||||
|
||||
return errors.Wrap(err, "error specializing function pod")
|
||||
}
|
||||
|
||||
return errors.Wrapf(err, "error specializing function pod after %v times", maxRetries)
|
||||
}
|
||||
Reference in New Issue
Block a user