diff --git a/builder/builder.go b/builder/builder.go index df5b8664..8ac9c895 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -19,11 +19,9 @@ package builder import ( "bufio" "encoding/json" - "errors" "fmt" "io" "io/ioutil" - "log" "net/http" "os" "os/exec" @@ -33,6 +31,8 @@ import ( "time" "github.com/dchest/uniuri" + "github.com/pkg/errors" + "go.uber.org/zap" "github.com/fission/fission" ) @@ -60,12 +60,14 @@ type ( } Builder struct { + logger *zap.Logger sharedVolumePath string } ) -func MakeBuilder(sharedVolumePath string) *Builder { +func MakeBuilder(logger *zap.Logger, sharedVolumePath string) *Builder { return &Builder{ + logger: logger.Named("builder"), sharedVolumePath: sharedVolumePath, } } @@ -77,37 +79,37 @@ func (builder *Builder) VersionHandler(w http.ResponseWriter, r *http.Request) { func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) { if r.Method != "POST" { - e := fmt.Sprintf("Method not allowed: %v", r.Method) - log.Println(e) - builder.reply(w, "", e, http.StatusMethodNotAllowed) + e := "method not allowed" + builder.logger.Error(e, zap.String("http_method", r.Method)) + builder.reply(w, "", fmt.Sprintf("%s: %s", e, r.Method), http.StatusMethodNotAllowed) return } startTime := time.Now() defer func() { elapsed := time.Since(startTime) - log.Printf("elapsed time in build request = %v", elapsed) + builder.logger.Info("build request complete", zap.Duration("elapsed_time", elapsed)) }() // parse request body, err := ioutil.ReadAll(r.Body) if err != nil { - e := errors.New(fmt.Sprintf("Error reading request body: %v", err)) - log.Println(e.Error()) - builder.reply(w, "", e.Error(), http.StatusInternalServerError) + e := "error reading request body" + builder.logger.Error(e, zap.Error(err)) + builder.reply(w, "", fmt.Sprintf("%s: %s", e, err.Error()), http.StatusInternalServerError) return } var req PackageBuildRequest err = json.Unmarshal(body, &req) if err != nil { - e := errors.New(fmt.Sprintf("Error parsing json body: %v", err)) - log.Println(e.Error()) - builder.reply(w, "", e.Error(), http.StatusBadRequest) + e := "error parsing json body" + builder.logger.Error(e, zap.Error(err)) + builder.reply(w, "", fmt.Sprintf("%s: %s", e, err.Error()), http.StatusBadRequest) return } - log.Printf("Builder received request: %v", req) + builder.logger.Info("builder received request", zap.Any("request", req)) - log.Println("Starting build...") + builder.logger.Info("starting build") srcPkgPath := filepath.Join(builder.sharedVolumePath, req.SrcPkgFilename) deployPkgFilename := fmt.Sprintf("%v-%v", req.SrcPkgFilename, strings.ToLower(uniuri.NewLen(6))) deployPkgPath := filepath.Join(builder.sharedVolumePath, deployPkgFilename) @@ -118,10 +120,11 @@ func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) { } buildLogs, err := builder.build(buildCmd, srcPkgPath, deployPkgPath) if err != nil { - e := errors.New(fmt.Sprintf("Error building source package: %v", err)) - log.Println(e.Error()) + e := "error building source package" + builder.logger.Error(e, zap.Error(err)) + // append error at the end of build logs - buildLogs += fmt.Sprintf("%v\n", e.Error()) + buildLogs += fmt.Sprintf("%s: %s\n", e, err.Error()) builder.reply(w, deployPkgFilename, buildLogs, http.StatusInternalServerError) return } @@ -137,7 +140,7 @@ func (builder *Builder) reply(w http.ResponseWriter, pkgFilename string, buildLo rBody, err := json.Marshal(resp) if err != nil { - e := errors.New(fmt.Sprintf("Error encoding response body: %v", err)) + e := errors.Wrap(err, "error encoding response body") rBody = []byte(fmt.Sprintf(`{"buildLogs": "%v"}`, e.Error())) statusCode = http.StatusInternalServerError } @@ -154,7 +157,7 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s fi, err := os.Stat(srcPkgPath) if err != nil { - return "", errors.New(fmt.Sprintf("could not find srcPkgPath: '%s'", srcPkgPath)) + return "", fmt.Errorf("could not find srcPkgPath: '%s'", srcPkgPath) } if fi.IsDir() { cmd.Dir = srcPkgPath @@ -170,12 +173,12 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s stdout, err := cmd.StdoutPipe() if err != nil { - return "", errors.New(fmt.Sprintf("Error creating stdout pipe for cmd: %v", err.Error())) + return "", errors.Wrap(err, "error creating stdout pipe for cmd") } stderr, err := cmd.StderrPipe() if err != nil { - return "", errors.New(fmt.Sprintf("Error creating stderr pipe for cmd: %v", err.Error())) + return "", errors.Wrap(err, "error creating stderr pipe for cmd") } var buildLogs string @@ -190,7 +193,7 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s err = cmd.Start() if err != nil { - return "", errors.New(fmt.Sprintf("Error starting cmd: %v", err.Error())) + return "", errors.Wrap(err, "error starting cmd") } // Runtime logs @@ -201,14 +204,14 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s } if err := scanner.Err(); err != nil { - scanErr := errors.New(fmt.Sprintf("Error reading cmd output: %v", err.Error())) + scanErr := errors.Wrap(err, "error reading cmd output") fmt.Println(scanErr) return buildLogs, scanErr } err = cmd.Wait() if err != nil { - cmdErr := errors.New(fmt.Sprintf("Error waiting for cmd '%v': %v", command, err.Error())) + cmdErr := errors.Wrapf(err, "error waiting for cmd %q", command) fmt.Println(cmdErr) return buildLogs, cmdErr } diff --git a/builder/client/client.go b/builder/client/client.go index 38525eec..4f1b7f76 100644 --- a/builder/client/client.go +++ b/builder/client/client.go @@ -20,31 +20,35 @@ import ( "bytes" "encoding/json" "io/ioutil" - "log" "net/http" "strings" "time" + "github.com/pkg/errors" + "go.uber.org/zap" + "github.com/fission/fission" builder "github.com/fission/fission/builder" ) type ( Client struct { - url string + logger *zap.Logger + url string } ) -func MakeClient(builderUrl string) *Client { +func MakeClient(logger *zap.Logger, builderUrl string) *Client { return &Client{ - url: strings.TrimSuffix(builderUrl, "/"), + logger: logger.Named("builder_client"), + url: strings.TrimSuffix(builderUrl, "/"), } } func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildResponse, error) { body, err := json.Marshal(req) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error marshaling json") } maxRetries := 20 @@ -62,7 +66,7 @@ func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildR if i < maxRetries-1 { time.Sleep(50 * time.Duration(2*i) * time.Millisecond) - log.Printf("Error building package (%v), retrying", err) + c.logger.Error("error building package, retrying", zap.Error(err)) continue } @@ -73,14 +77,14 @@ func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildR rBody, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Printf("Error reading resp body: %v", err) + c.logger.Error("error reading resp body", zap.Error(err)) return nil, err } pkgBuildResp := builder.PackageBuildResponse{} err = json.Unmarshal([]byte(rBody), &pkgBuildResp) if err != nil { - log.Printf("Error parsing resp body: %v", err) + c.logger.Error("error parsing resp body", zap.Error(err)) return nil, err } diff --git a/builder/cmd/main.go b/builder/cmd/main.go index e22f6d26..5e3e852e 100644 --- a/builder/cmd/main.go +++ b/builder/cmd/main.go @@ -21,21 +21,29 @@ import ( "net/http" "os" + "go.uber.org/zap" + builder "github.com/fission/fission/builder" ) // Usage: builder func main() { + logger, err := zap.NewProduction() + if err != nil { + log.Fatalf("can't initialize zap logger: %v", err) + } + defer logger.Sync() + dir := os.Args[1] if _, err := os.Stat(dir); err != nil { if os.IsNotExist(err) { err = os.MkdirAll(dir, os.ModeDir|0700) if err != nil { - log.Fatalf("Error creating directory: %v", err) + logger.Fatal("error creating directory", zap.Error(err), zap.String("directory", dir)) } } } - builder := builder.MakeBuilder(dir) + builder := builder.MakeBuilder(logger, dir) mux := http.NewServeMux() mux.HandleFunc("/", builder.Handler) mux.HandleFunc("/version", builder.VersionHandler) diff --git a/buildermgr/buildermgr.go b/buildermgr/buildermgr.go index ebedac75..2798dbc3 100644 --- a/buildermgr/buildermgr.go +++ b/buildermgr/buildermgr.go @@ -17,29 +17,30 @@ limitations under the License. package buildermgr import ( - "log" + "github.com/pkg/errors" + "go.uber.org/zap" "github.com/fission/fission/crd" ) // Start the buildermgr service. -func Start(storageSvcUrl string, envBuilderNamespace string) error { +func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error { + bmLogger := logger.Named("builder_manager") fissionClient, kubernetesClient, _, err := crd.MakeFissionClient() if err != nil { - log.Printf("Failed to get kubernetes client: %v", err) - return err + return errors.Wrap(err, "failed to get fission or kubernetes client") } err = fissionClient.WaitForCRDs() if err != nil { - log.Fatalf("Error waiting for CRDs: %v", err) + return errors.Wrap(err, "error waiting for CRDs") } - envWatcher := makeEnvironmentWatcher(fissionClient, kubernetesClient, envBuilderNamespace) + envWatcher := makeEnvironmentWatcher(bmLogger, fissionClient, kubernetesClient, envBuilderNamespace) go envWatcher.watchEnvironments() - pkgWatcher := makePackageWatcher(fissionClient, + pkgWatcher := makePackageWatcher(bmLogger, fissionClient, kubernetesClient, envBuilderNamespace, storageSvcUrl) go pkgWatcher.watchPackages(fissionClient, kubernetesClient, envBuilderNamespace) diff --git a/buildermgr/common.go b/buildermgr/common.go index 2f24182c..8b0a7178 100644 --- a/buildermgr/common.go +++ b/buildermgr/common.go @@ -19,11 +19,13 @@ package buildermgr import ( "context" "fmt" - "log" "net/http" "strings" "github.com/dchest/uniuri" + "github.com/pkg/errors" + "go.uber.org/zap" + "github.com/fission/fission" "github.com/fission/fission/builder" builderClient "github.com/fission/fission/builder/client" @@ -38,20 +40,21 @@ import ( // 3. Send upload request to fetcher to upload deployment package. // 4. Return upload response and build logs. // *. Return build logs and error if any one of steps above failed. -func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuilderNamespace string, +func buildPackage(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, envBuilderNamespace string, storageSvcUrl string, pkg *crd.Package) (uploadResp *fission.ArchiveUploadResponse, buildLogs string, err error) { env, err := fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name) if err != nil { - e := fmt.Sprintf("Error getting environment CRD info: %v", err) - log.Println(e) + e := "error getting environment CRD info" + logger.Error(e, zap.Error(err)) + e = fmt.Sprintf("%s: %v", e, err) return nil, e, fission.MakeError(http.StatusInternalServerError, e) } svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, envBuilderNamespace) srcPkgFilename := fmt.Sprintf("%v-%v", pkg.Metadata.Name, strings.ToLower(uniuri.NewLen(6))) - fetcherC := fetcherClient.MakeClient(fmt.Sprintf("http://%v:8000", svcName)) - builderC := builderClient.MakeClient(fmt.Sprintf("http://%v:8001", svcName)) + fetcherC := fetcherClient.MakeClient(logger, fmt.Sprintf("http://%v:8000", svcName)) + builderC := builderClient.MakeClient(logger, fmt.Sprintf("http://%v:8001", svcName)) fetchReq := &fission.FunctionFetchRequest{ FetchType: fission.FETCH_SOURCE, @@ -63,8 +66,9 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil // send fetch request to fetcher err = fetcherC.Fetch(ctx, fetchReq) if err != nil { - e := fmt.Sprintf("Error fetching source package: %v", err) - log.Println(e) + e := "error fetching source package" + logger.Error(e, zap.Error(err)) + e = fmt.Sprintf("%s: %v", e, err) return nil, e, fission.MakeError(http.StatusInternalServerError, e) } @@ -78,12 +82,11 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil BuildCommand: buildCmd, } - log.Printf("Start building with source package: %v", srcPkgFilename) + logger.Info("started building with source package", zap.String("source_package", srcPkgFilename)) // send build request to builder buildResp, err := builderC.Build(pkgBuildReq) if err != nil { e := fmt.Sprintf("Error building deployment package: %v", err) - log.Println(e) var buildLogs string if buildResp != nil { buildLogs = buildResp.BuildLogs @@ -92,7 +95,7 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil return nil, buildLogs, fission.MakeError(http.StatusInternalServerError, e) } - log.Printf("Build succeed, source package: %v, deployment package: %v", srcPkgFilename, buildResp.ArtifactFilename) + logger.Info("build succeed", zap.String("source_package", srcPkgFilename), zap.String("deployment_package", buildResp.ArtifactFilename)) archivePackage := !env.Spec.KeepArchive @@ -102,12 +105,11 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil ArchivePackage: archivePackage, } - log.Printf("Start uploading deployment package: %v", buildResp.ArtifactFilename) + logger.Info("started uploading deployment package", zap.String("deployment_package", buildResp.ArtifactFilename)) // ask fetcher to upload the deployment package uploadResp, err = fetcherC.Upload(ctx, uploadReq) if err != nil { e := fmt.Sprintf("Error uploading deployment package: %v", err) - log.Println(e) buildResp.BuildLogs += fmt.Sprintf("%v\n", e) return nil, buildResp.BuildLogs, fission.MakeError(http.StatusInternalServerError, e) } @@ -115,7 +117,7 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil return uploadResp, buildResp.BuildLogs, nil } -func updatePackage(fissionClient *crd.FissionClient, +func updatePackage(logger *zap.Logger, fissionClient *crd.FissionClient, pkg *crd.Package, status fission.BuildStatus, buildLogs string, uploadResp *fission.ArchiveUploadResponse) (*crd.Package, error) { @@ -135,8 +137,9 @@ func updatePackage(fissionClient *crd.FissionClient, // update package spec pkg, err := fissionClient.Packages(pkg.Metadata.Namespace).Update(pkg) if err != nil { - log.Printf("Error updating package: %v", err) - return nil, err + e := "error updating package" + logger.Error(e, zap.Error(err)) + return nil, errors.Wrap(err, e) } // return resource version for function to update function package ref diff --git a/buildermgr/envwatcher.go b/buildermgr/envwatcher.go index 571547c7..6d879db7 100644 --- a/buildermgr/envwatcher.go +++ b/buildermgr/envwatcher.go @@ -18,11 +18,12 @@ package buildermgr import ( "fmt" - "log" "os" "strconv" "time" + "github.com/pkg/errors" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -73,6 +74,7 @@ type ( } environmentWatcher struct { + logger *zap.Logger cache map[string]*builderInfo requestChan chan envwRequest builderNamespace string @@ -86,7 +88,7 @@ type ( } ) -func makeEnvironmentWatcher(fissionClient *crd.FissionClient, +func makeEnvironmentWatcher(logger *zap.Logger, fissionClient *crd.FissionClient, kubernetesClient *kubernetes.Clientset, builderNamespace string) *environmentWatcher { useIstio := false @@ -94,7 +96,7 @@ func makeEnvironmentWatcher(fissionClient *crd.FissionClient, if len(enableIstio) > 0 { istio, err := strconv.ParseBool(enableIstio) if err != nil { - log.Println("Failed to parse ENABLE_ISTIO, defaults to false") + logger.Info("Failed to parse ENABLE_ISTIO, defaults to false") } useIstio = istio } @@ -109,6 +111,7 @@ func makeEnvironmentWatcher(fissionClient *crd.FissionClient, collectorEndpoint := os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT") envWatcher := &environmentWatcher{ + logger: logger.Named("environment_watcher"), cache: make(map[string]*builderInfo), requestChan: make(chan envwRequest), builderNamespace: builderNamespace, @@ -153,11 +156,11 @@ func (envw *environmentWatcher) watchEnvironments() { }) if err != nil { if fission.IsNetworkError(err) { - log.Printf("Encounter network error, retrying later: %v", err) + envw.logger.Error("encountered network error, retrying later", zap.Error(err)) time.Sleep(5 * time.Second) continue } - log.Fatalf("Error watching environment list: %v", err) + envw.logger.Fatal("error watching environment list", zap.Error(err)) } for { @@ -185,11 +188,11 @@ func (envw *environmentWatcher) sync() { envList, err := envw.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { if fission.IsNetworkError(err) { - log.Printf("Error syncing environment CRD resources due to network error, retrying later: %v", err) + envw.logger.Error("error syncing environment CRD resources due to network error, retrying later", zap.Error(err)) time.Sleep(50 * time.Duration(2*i) * time.Millisecond) continue } - log.Fatalf("Error syncing environment CRD resources: %v", err) + envw.logger.Fatal("error syncing environment CRD resources", zap.Error(err)) } // Create environment builders for all environments @@ -202,7 +205,7 @@ func (envw *environmentWatcher) sync() { } _, err := envw.getEnvBuilder(&env) if err != nil { - log.Printf("Error creating builder for %v: %v", env.Metadata.Name, err) + envw.logger.Error("error creating builder", zap.Error(err), zap.String("builder_target", env.Metadata.Name)) } } @@ -258,7 +261,7 @@ func (envw *environmentWatcher) service() { svcList, err := envw.getBuilderServiceList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll) if err != nil { - log.Println(err.Error()) + envw.logger.Error("error getting the builder service list", zap.Error(err)) } for _, svc := range svcList { envName := svc.ObjectMeta.Labels[LABEL_ENV_NAME] @@ -268,7 +271,9 @@ func (envw *environmentWatcher) service() { if _, ok := latestEnvList[key]; !ok { err := envw.deleteBuilderServiceByName(svc.ObjectMeta.Name, svc.ObjectMeta.Namespace) if err != nil { - log.Printf("Error removing builder service: %v", err) + envw.logger.Error("error removing builder service", zap.Error(err), + zap.String("service_name", svc.ObjectMeta.Name), + zap.String("service_namespace", svc.ObjectMeta.Namespace)) } } delete(envw.cache, key) @@ -276,7 +281,7 @@ func (envw *environmentWatcher) service() { deployList, err := envw.getBuilderDeploymentList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll) if err != nil { - log.Printf(err.Error()) + envw.logger.Error("error getting the builder deployment list", zap.Error(err)) } for _, deploy := range deployList { envName := deploy.ObjectMeta.Labels[LABEL_ENV_NAME] @@ -286,7 +291,9 @@ func (envw *environmentWatcher) service() { if _, ok := latestEnvList[key]; !ok { err := envw.deleteBuilderDeploymentByName(deploy.ObjectMeta.Name, deploy.ObjectMeta.Namespace) if err != nil { - log.Printf("Error removing builder deployment: %v", err) + envw.logger.Error("error removing builder deployment", zap.Error(err), + zap.String("deployment_name", deploy.ObjectMeta.Name), + zap.String("deployment_namespace", deploy.ObjectMeta.Namespace)) } } delete(envw.cache, key) @@ -327,12 +334,12 @@ func (envw *environmentWatcher) createBuilder(env *crd.Environment, ns string) ( if len(svcList) == 0 { svc, err = envw.createBuilderService(env, ns) if err != nil { - return nil, fmt.Errorf("Error creating builder service: %v", err) + return nil, errors.Wrap(err, "error creating builder service") } } else if len(svcList) == 1 { svc = &svcList[0] } else { - return nil, fmt.Errorf("Found more than one builder service for environment %v", env.Metadata.Name) + return nil, fmt.Errorf("found more than one builder service for environment %q", env.Metadata.Name) } deployList, err := envw.getBuilderDeploymentList(sel, ns) @@ -344,18 +351,17 @@ func (envw *environmentWatcher) createBuilder(env *crd.Environment, ns string) ( // create builder SA in this ns, if not already created _, err := fission.SetupSA(envw.kubernetesClient, fission.FissionBuilderSA, ns) if err != nil { - log.Printf("Error : %v creating %s in ns : %s", err, fission.FissionBuilderSA, ns) - return nil, err + return nil, errors.Wrapf(err, "error creating %q in ns: %s", fission.FissionBuilderSA, ns) } deploy, err = envw.createBuilderDeployment(env, ns) if err != nil { - return nil, fmt.Errorf("Error creating builder deployment: %v", err) + return nil, errors.Wrap(err, "error creating builder deployment") } } else if len(deployList) == 1 { deploy = &deployList[0] } else { - return nil, fmt.Errorf("Found more than one builder deployment for environment %v", env.Metadata.Name) + return nil, fmt.Errorf("found more than one builder deployment for environment %q", env.Metadata.Name) } return &builderInfo{ @@ -370,7 +376,7 @@ func (envw *environmentWatcher) deleteBuilderServiceByName(name, namespace strin Services(namespace). Delete(name, &delOpt) if err != nil { - return fmt.Errorf("Error deleting builder service %s.%s: %v", name, namespace, err) + return errors.Wrapf(err, "error deleting builder service %s.%s", name, namespace) } return nil } @@ -380,7 +386,7 @@ func (envw *environmentWatcher) deleteBuilderDeploymentByName(name, namespace st Deployments(namespace). Delete(name, &delOpt) if err != nil { - return fmt.Errorf("Error deleting builder deployment %s.%s: %v", name, namespace, err) + return errors.Wrapf(err, "error deleting builder deployment %s.%s", name, namespace) } return nil } @@ -391,12 +397,12 @@ func (envw *environmentWatcher) deleteBuilderService(sel map[string]string, ns s return err } for _, svc := range svcList { - log.Printf("Removing builder service: %v", svc.ObjectMeta.Name) + envw.logger.Info("removing builder service", zap.String("service_name", svc.ObjectMeta.Name)) err = envw.kubernetesClient.CoreV1(). Services(ns). Delete(svc.ObjectMeta.Name, &delOpt) if err != nil { - return fmt.Errorf("Error deleting builder service: %v", err) + return errors.Wrap(err, "error deleting builder service") } } return nil @@ -408,12 +414,12 @@ func (envw *environmentWatcher) deleteBuilderDeployment(sel map[string]string, n return err } for _, deploy := range deployList { - log.Printf("Removing builder deployment: %v", deploy.ObjectMeta.Name) + envw.logger.Info("removing builder deployment", zap.String("deployment_name", deploy.ObjectMeta.Name)) err = envw.kubernetesClient.ExtensionsV1beta1(). Deployments(ns). Delete(deploy.ObjectMeta.Name, &delOpt) if err != nil { - return fmt.Errorf("Error deleteing builder deployment: %v", err) + return errors.Wrap(err, "error deleteing builder deployment") } } return nil @@ -425,7 +431,7 @@ func (envw *environmentWatcher) getBuilderServiceList(sel map[string]string, ns LabelSelector: labels.Set(sel).AsSelector().String(), }) if err != nil { - return nil, fmt.Errorf("Error getting builder service list: %v", err) + return nil, errors.Wrap(err, "error getting builder service list") } return svcList.Items, nil } @@ -464,7 +470,7 @@ func (envw *environmentWatcher) createBuilderService(env *crd.Environment, ns st }, }, } - log.Printf("Creating builder service: %v", name) + envw.logger.Info("creating builder service", zap.String("service_name", name)) _, err := envw.kubernetesClient.CoreV1().Services(ns).Create(&service) if err != nil { return nil, err @@ -478,7 +484,7 @@ func (envw *environmentWatcher) getBuilderDeploymentList(sel map[string]string, LabelSelector: labels.Set(sel).AsSelector().String(), }) if err != nil { - return nil, fmt.Errorf("Error getting builder deployment list: %v", err) + return nil, errors.Wrap(err, "error getting builder deployment list") } return deployList.Items, nil } @@ -615,7 +621,7 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns }, }, } - log.Printf("Creating builder deployment: %v", fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion)) + envw.logger.Info("creating builder deployment", zap.String("deployment", name)) _, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Create(deployment) if err != nil { return nil, err diff --git a/buildermgr/pkgwatcher.go b/buildermgr/pkgwatcher.go index 21b1444d..2c4d869d 100644 --- a/buildermgr/pkgwatcher.go +++ b/buildermgr/pkgwatcher.go @@ -19,23 +19,24 @@ package buildermgr import ( "context" "fmt" - "log" "time" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" k8sCache "k8s.io/client-go/tools/cache" "github.com/fission/fission" "github.com/fission/fission/cache" "github.com/fission/fission/crd" - "k8s.io/client-go/kubernetes" ) type ( packageWatcher struct { + logger *zap.Logger fissionClient *crd.FissionClient k8sClient *kubernetes.Clientset podStore k8sCache.Store @@ -45,13 +46,14 @@ type ( } ) -func makePackageWatcher(fissionClient *crd.FissionClient, k8sClientSet *kubernetes.Clientset, +func makePackageWatcher(logger *zap.Logger, fissionClient *crd.FissionClient, k8sClientSet *kubernetes.Clientset, builderNamespace string, storageSvcUrl string) *packageWatcher { lw := k8sCache.NewListWatchFromClient(k8sClientSet.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.Everything()) store, controller := k8sCache.NewInformer(lw, &apiv1.Pod{}, 30*time.Second, k8sCache.ResourceEventHandlerFuncs{}) go controller.Run(make(chan struct{})) pkgw := &packageWatcher{ + logger: logger.Named("package_watcher"), fissionClient: fissionClient, k8sClient: k8sClientSet, podStore: store, @@ -86,20 +88,20 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package) } defer buildCache.Delete(key) - log.Printf("Start build for package %v with resource version %v", srcpkg.Metadata.Name, srcpkg.Metadata.ResourceVersion) + pkgw.logger.Info("starting build for package", zap.String("package_name", srcpkg.Metadata.Name), zap.String("resource_version", srcpkg.Metadata.ResourceVersion)) - pkg, err := updatePackage(pkgw.fissionClient, srcpkg, fission.BuildStatusRunning, "", nil) + pkg, err := updatePackage(pkgw.logger, pkgw.fissionClient, srcpkg, fission.BuildStatusRunning, "", nil) if err != nil { - e := fmt.Sprintf("Error setting package pending state: %v", err) - log.Println(e) - updatePackage(pkgw.fissionClient, srcpkg, fission.BuildStatusFailed, e, nil) + pkgw.logger.Error("error setting package pending state", zap.Error(err)) return } env, err := pkgw.fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name) - if errors.IsNotFound(err) { - updatePackage(pkgw.fissionClient, pkg, - fission.BuildStatusFailed, "Environment not existed", nil) + if k8serrors.IsNotFound(err) { + e := "environment does not exist" + pkgw.logger.Error(e, zap.String("environment", pkg.Spec.Environment.Name)) + updatePackage(pkgw.logger, pkgw.fissionClient, pkg, + fission.BuildStatusFailed, fmt.Sprintf("%s: %q", e, pkg.Spec.Environment.Name), nil) return } @@ -109,12 +111,12 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package) // iterate all available environment builders. items := pkgw.podStore.List() if err != nil { - log.Printf("Error retrieving pod information for env %v: %v", err, env.Metadata.Name) + pkgw.logger.Error("error retrieving pod information for environment", zap.Error(err), zap.String("environment", env.Metadata.Name)) return } if len(items) == 0 { - log.Printf("Environment \"%v\" builder pod is not existed yet, retry again later.", pkg.Spec.Environment.Name) + pkgw.logger.Info("builder pod does not exist for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name)) time.Sleep(time.Duration(i*1) * time.Second) continue } @@ -145,7 +147,7 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package) } if !podIsReady { - log.Printf("Environment \"%v\" builder pod is not ready, retry again later.", pkg.Spec.Environment.Name) + pkgw.logger.Info("builder pod is not ready for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name)) time.Sleep(time.Duration(i*1) * time.Second) break } @@ -153,31 +155,37 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package) // Add the package getter rolebinding to builder sa // we continue here if role binding was not setup succeesffully. this is because without this, the fetcher wont be able to fetch the source pkg into the container and // the build will fail eventually - err := fission.SetupRoleBinding(pkgw.k8sClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, builderNs) + err := fission.SetupRoleBinding(pkgw.logger, pkgw.k8sClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, builderNs) if err != nil { - log.Printf("Error : %v in setting up the role binding %s for pkg : %s.%s", err, fission.PackageGetterRB, pkg.Metadata.Name, pkg.Metadata.Namespace) + pkgw.logger.Error("error setting up role binding for package", + zap.Error(err), + zap.String("role_binding", fission.PackageGetterRB), + zap.String("package_name", pkg.Metadata.Name), + zap.String("package_namespace", pkg.Metadata.Namespace)) continue } else { - log.Printf("Setup rolebinding for sa : %s.%s for pkg : %s.%s", fission.FissionBuilderSA, builderNs, pkg.Metadata.Name, pkg.Metadata.Namespace) + pkgw.logger.Info("setup rolebinding for sa package", + zap.String("sa", fmt.Sprintf("%s.%s", fission.FissionBuilderSA, builderNs)), + zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace))) } ctx := context.Background() - uploadResp, buildLogs, err := buildPackage(ctx, pkgw.fissionClient, builderNs, pkgw.storageSvcUrl, pkg) + uploadResp, buildLogs, err := buildPackage(ctx, pkgw.logger, pkgw.fissionClient, builderNs, pkgw.storageSvcUrl, pkg) if err != nil { - log.Printf("Error building package %v: %v", pkg.Metadata.Name, err) - updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) + pkgw.logger.Error("error building package", zap.Error(err), zap.String("package_name", pkg.Metadata.Name)) + updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) return } - log.Printf("Start updating info of package: %v", pkg.Metadata.Name) + pkgw.logger.Info("starting package info update", zap.String("package_name", pkg.Metadata.Name)) fnList, err := pkgw.fissionClient. Functions(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { - e := fmt.Sprintf("Error getting function list: %v", err) - log.Println(e) - buildLogs += fmt.Sprintf("%v\n", e) - updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) + e := "error getting function list" + pkgw.logger.Error(e, zap.Error(err)) + buildLogs += fmt.Sprintf("%s: %v\n", e, err) + updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) } // A package may be used by multiple functions. Update @@ -190,33 +198,33 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package) // update CRD _, err = pkgw.fissionClient.Functions(fn.Metadata.Namespace).Update(&fn) if err != nil { - e := fmt.Sprintf("Error updating function package resource version: %v", err) - log.Println(e) - buildLogs += fmt.Sprintf("%v\n", e) - updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) + e := "error updating function package resource version" + pkgw.logger.Error(e, zap.Error(err)) + buildLogs += fmt.Sprintf("%s: %v\n", e, err) + updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) return } } } - _, err = updatePackage(pkgw.fissionClient, pkg, + _, err = updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusSucceeded, buildLogs, uploadResp) if err != nil { - log.Printf("Error update package info: %v", err) - updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) + pkgw.logger.Error("error updating package info", zap.Error(err), zap.String("package_name", pkg.Metadata.Name)) + updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) return } - log.Printf("Completed build request for package: %v", pkg.Metadata.Name) + pkgw.logger.Info("completed package build request", zap.String("package_name", pkg.Metadata.Name)) return } } // build timeout - updatePackage(pkgw.fissionClient, pkg, + updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, "Build timeout due to environment builder not ready", nil) - log.Printf("Max retries exceeded in building the source pkg : %s.%s, timeout due to environment builder not ready", - pkg.Metadata.Name, pkg.Metadata.Namespace) + pkgw.logger.Error("max retries exceeded in building source package, timeout due to environment builder not ready", + zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace))) return } diff --git a/canaryconfigmgr/canaryConfigCache.go b/canaryconfigmgr/canaryConfigCache.go index 793fe012..6d5c828f 100644 --- a/canaryconfigmgr/canaryConfigCache.go +++ b/canaryconfigmgr/canaryConfigCache.go @@ -18,11 +18,11 @@ package canaryconfigmgr import ( "context" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission/cache" - "time" ) type ( diff --git a/canaryconfigmgr/canaryConfigMgr.go b/canaryconfigmgr/canaryConfigMgr.go index 73e9fd97..1c5fb386 100644 --- a/canaryconfigmgr/canaryConfigMgr.go +++ b/canaryconfigmgr/canaryConfigMgr.go @@ -24,7 +24,8 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" + "github.com/pkg/errors" + "go.uber.org/zap" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -37,6 +38,7 @@ import ( ) type canaryConfigMgr struct { + logger *zap.Logger fissionClient *crd.FissionClient kubeClient *kubernetes.Clientset canaryConfigStore k8sCache.Store @@ -46,9 +48,9 @@ type canaryConfigMgr struct { canaryCfgCancelFuncMap *canaryConfigCancelFuncMap } -func MakeCanaryConfigMgr(fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, crdClient *rest.RESTClient, prometheusSvc string) (*canaryConfigMgr, error) { +func MakeCanaryConfigMgr(logger *zap.Logger, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, crdClient *rest.RESTClient, prometheusSvc string) (*canaryConfigMgr, error) { if prometheusSvc == "" { - log.Info("Try to retrieve prometheus server information from environment variables") + logger.Info("try to retrieve prometheus server information from environment variables") var prometheusSvcHost, prometheusSvcPort string // handle a case where there is a prometheus server is already installed, try to find the service from env variable @@ -73,12 +75,13 @@ func MakeCanaryConfigMgr(fissionClient *crd.FissionClient, kubeClient *kubernete return nil, fmt.Errorf("prometheus service url not found/invalid, cant create canary config manager: %v", prometheusSvc) } - promClient, err := MakePrometheusClient(prometheusSvc) + promClient, err := MakePrometheusClient(logger, prometheusSvc) if err != nil { return nil, err } configMgr := &canaryConfigMgr{ + logger: logger.Named("canary_config_manager"), fissionClient: fissionClient, kubeClient: kubeClient, crdClient: crdClient, @@ -113,7 +116,10 @@ func (canaryCfgMgr *canaryConfigMgr) initCanaryConfigController() (k8sCache.Stor newConfig := newObj.(*crd.CanaryConfig) if oldConfig.Metadata.ResourceVersion != newConfig.Metadata.ResourceVersion && newConfig.Status.Status == fission.CanaryConfigStatusPending { - log.Printf("update canary config invoked for : %s.%s, newConfig.Status.Status=%s", newConfig.Metadata.Name, newConfig.Metadata.Namespace, newConfig.Status.Status) + canaryCfgMgr.logger.Info("update canary config invoked", + zap.String("name", newConfig.Metadata.Name), + zap.String("namespace", newConfig.Metadata.Namespace), + zap.String("version", newConfig.Metadata.ResourceVersion)) go canaryCfgMgr.updateCanaryConfig(oldConfig, newConfig) } go canaryCfgMgr.reSyncCanaryConfigs() @@ -126,17 +132,21 @@ func (canaryCfgMgr *canaryConfigMgr) initCanaryConfigController() (k8sCache.Stor func (canaryCfgMgr *canaryConfigMgr) Run(ctx context.Context) { go canaryCfgMgr.canaryConfigController.Run(ctx.Done()) - log.Printf("started Canary configmgr controller") + canaryCfgMgr.logger.Info("started canary configmgr controller") } func (canaryCfgMgr *canaryConfigMgr) addCanaryConfig(canaryConfig *crd.CanaryConfig) { - log.Printf("addCanaryConfig called for %s", canaryConfig.Metadata.Name) + canaryCfgMgr.logger.Info("addCanaryConfig called", zap.String("canary_config", canaryConfig.Metadata.Name)) // for each canary config, create a ticker with increment interval interval, err := time.ParseDuration(canaryConfig.Spec.WeightIncrementDuration) if err != nil { - log.Printf("Error parsing duration: %v, cant proceed with this canaryConfig : %v.%v", err, - canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace) + canaryCfgMgr.logger.Error("error parsing duration - cant proceed with this canaryConfig", + zap.Error(err), + zap.String("duration", canaryConfig.Spec.WeightIncrementDuration), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) return } ticker := time.NewTicker(interval) @@ -151,7 +161,11 @@ func (canaryCfgMgr *canaryConfigMgr) addCanaryConfig(canaryConfig *crd.CanaryCon } err = canaryCfgMgr.canaryCfgCancelFuncMap.assign(&canaryConfig.Metadata, cacheValue) if err != nil { - log.Printf("Error caching canary config : %s.%s. err : %v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, err) + canaryCfgMgr.logger.Error("error caching canary config", + zap.Error(err), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) return } canaryCfgMgr.processCanaryConfig(&ctx, canaryConfig, ticker) @@ -164,10 +178,17 @@ func (canaryCfgMgr *canaryConfigMgr) processCanaryConfig(ctx *context.Context, c select { case <-(*ctx).Done(): // this case when someone deleted their canary config in the middle of it being processed - log.Printf("Cancel Func called for canary config : %s", canaryConfig.Metadata.Name) + canaryCfgMgr.logger.Info("cancel func called for canary config", + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata) if err != nil { - log.Printf("error removing canary config: %s from map, err : %v", canaryConfig.Metadata.Name, err) + canaryCfgMgr.logger.Error("error removing canary config", + zap.Error(err), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) } return @@ -175,16 +196,26 @@ func (canaryCfgMgr *canaryConfigMgr) processCanaryConfig(ctx *context.Context, c // every weightIncrementDuration, check if failureThreshold has reached. // if yes, rollback. // else, increment the weight of new function and decrement old function by `weightIncrement` - log.Printf("Processing canary config : %s.%s", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace) + canaryCfgMgr.logger.Info("processing canary config", + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) canaryCfgMgr.RollForwardOrBack(canaryConfig, quit, ticker) case <-quit: // we're done processing this canary config either because the new function receives 100% of the traffic // or we rolled back to send all 100% traffic to old function - log.Printf("Quit processing canaryConfig : %s", canaryConfig.Metadata.Name) + canaryCfgMgr.logger.Info("quit processing canaryConfig", + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata) if err != nil { - log.Printf("error removing canary config: %s from map, err : %v", canaryConfig.Metadata.Name, err) + canaryCfgMgr.logger.Error("error removing canary config from map", + zap.Error(err), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) } return } @@ -195,7 +226,10 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC // handle race between delete event and notification on ticker.C _, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata) if err != nil { - log.Printf("No need of processing the config, not in cache anymore") + canaryCfgMgr.logger.Info("no need of processing the config, not in cache anymore", + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) return } @@ -204,19 +238,31 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC if err != nil { // if the http trigger is not found, then give up processing this config. if k8serrors.IsNotFound(err) { - log.Printf("Http trigger object : %v.%v missing", canaryConfig.Spec.Trigger, canaryConfig.Metadata.Namespace) + canaryCfgMgr.logger.Error("http trigger object for canary config missing", + zap.Error(err), + zap.String("trigger", canaryConfig.Spec.Trigger), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) close(quit) return } // just silently ignore. wait for next window to increment weight - log.Printf("Error fetching http trigger object, err : %v", err) + canaryCfgMgr.logger.Error("error fetching http trigger object for config", + zap.Error(err), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) return } // handle a race between ticker.Stop and receiving a notification on ticker.C if canaryConfig.Status.Status != fission.CanaryConfigStatusPending { - log.Printf("No need of processing the config, not pending anymore") + canaryCfgMgr.logger.Info("no need of processing the config, not pending anymore", + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) return } @@ -227,22 +273,42 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC if err != nil { // silently ignore. wait for next window to increment weight - log.Printf("Error calculating failure percentage, err : %v", err) + canaryCfgMgr.logger.Info("error calculating failure percentage", + zap.Error(err), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) return } - log.Printf("Failure percentage calculated : %v for canaryConfig %s", failurePercent, canaryConfig.Metadata.Name) + canaryCfgMgr.logger.Info("failure percentage calculated for canaryConfig", + zap.Float64("failure_percent", failurePercent), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) + if failurePercent == -1 { // this means there were no requests triggered to this url during this window. return here and check back // during next iteration - log.Printf("Total requests received for url : %v is 0", triggerObj.Spec.RelativeURL) + canaryCfgMgr.logger.Info("total requests received for url is 0", zap.String("url", triggerObj.Spec.RelativeURL)) return } if int(failurePercent) > canaryConfig.Spec.FailureThreshold { - log.Printf("Failure percent %v crossed the threshold %v, so rolling back", failurePercent, canaryConfig.Spec.FailureThreshold) + canaryCfgMgr.logger.Error("failure percent crossed the threshold, so rolling back", + zap.Float64("failure_percent", failurePercent), + zap.Int("threshold", canaryConfig.Spec.FailureThreshold), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) ticker.Stop() - canaryCfgMgr.rollback(canaryConfig, triggerObj) + err := canaryCfgMgr.rollback(canaryConfig, triggerObj) + if err != nil { + canaryCfgMgr.logger.Error("error rolling back canary config", + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) + } close(quit) return } @@ -251,7 +317,12 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC doneProcessingCanaryConfig, err := canaryCfgMgr.rollForward(canaryConfig, triggerObj) if err != nil { // just log the error and hope that next iteration will succeed - log.Printf("Error incrementing weights for triggerObj : %v, err : %v", triggerObj.Metadata.Name, err) + canaryCfgMgr.logger.Error("error incrementing weights for trigger", + zap.Error(err), + zap.String("trigger", triggerObj.Metadata.Name), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) return } @@ -263,11 +334,17 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC fission.CanaryConfigStatusSucceeded) if err != nil { // cant do much after max retries other than logging it. - log.Printf("Error updating canary config : %s.%s after max retries, err :%v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, - err) + canaryCfgMgr.logger.Error("error updating canary config after max retries", + zap.Error(err), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) } - log.Printf("We're done processing canary config : %s. The new function is receiving all the traffic", canaryConfig.Metadata.Name) + canaryCfgMgr.logger.Info("done processing canary config - the new function is receiving all the traffic", + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) close(quit) return } @@ -277,8 +354,9 @@ func (canaryCfgMgr *canaryConfigMgr) updateHttpTriggerWithRetries(triggerName, t for i := 0; i < fission.MaxRetries; i++ { triggerObj, err := canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Get(triggerName) if err != nil { - log.Printf("Error getting http trigger object : %v", err) - return err + e := "error getting http trigger object" + canaryCfgMgr.logger.Error(e, zap.Error(err), zap.String("trigger_name", triggerName), zap.String("trigger_namespace", triggerNamespace)) + return errors.Wrap(err, e) } triggerObj.Spec.FunctionReference.FunctionWeights = fnWeights @@ -286,14 +364,21 @@ func (canaryCfgMgr *canaryConfigMgr) updateHttpTriggerWithRetries(triggerName, t _, err = canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Update(triggerObj) switch { case err == nil: - log.Printf("Updated Http trigger : %s.%s", triggerName, triggerNamespace) + canaryCfgMgr.logger.Info("updated http trigger", zap.String("trigger_name", triggerName), zap.String("trigger_namespace", triggerNamespace)) return nil case k8serrors.IsConflict(err): - log.Printf("Conflict in updating http trigger : %s.%s, retrying", triggerName, triggerNamespace) + canaryCfgMgr.logger.Info("conflict in updating http trigger, retrying", + zap.Error(err), + zap.String("trigger_name", triggerName), + zap.String("trigger_namespace", triggerNamespace)) continue default: - log.Printf("Error updating trigger : %s.%s = %v", triggerName, triggerNamespace, err) - return err + e := "error updating http trigger" + canaryCfgMgr.logger.Info(e, + zap.Error(err), + zap.String("trigger_name", triggerName), + zap.String("trigger_namespace", triggerNamespace)) + return errors.Wrapf(err, "%s: %s.%s", e, triggerName, triggerNamespace) } } @@ -304,24 +389,42 @@ func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfigStatusWithRetries(cfgName for i := 0; i < fission.MaxRetries; i++ { canaryCfgObj, err := canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Get(cfgName) if err != nil { - log.Printf("Error getting http Canary Config object : %v", err) - return err + e := "error getting http canary config object" + canaryCfgMgr.logger.Error(e, + zap.Error(err), + zap.String("name", cfgName), + zap.String("namespace", cfgNamespace), + zap.String("status", status)) + return errors.Wrap(err, e) } - log.Printf("Updating status of canaryCfg : %s.%s to %s", cfgName, cfgNamespace, status) + canaryCfgMgr.logger.Info("updating status of canary config", + zap.String("name", cfgName), + zap.String("namespace", cfgNamespace), + zap.String("status", status)) + canaryCfgObj.Status.Status = status _, err = canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Update(canaryCfgObj) switch { case err == nil: - log.Printf("Updated Canary Config : %s.%s", cfgName, cfgNamespace) + canaryCfgMgr.logger.Info("updated canary config", + zap.String("name", cfgName), + zap.String("namespace", cfgNamespace)) return nil case k8serrors.IsConflict(err): - log.Printf("Conflict in updating Canary Config : %s.%s, retrying", cfgName, cfgNamespace) + canaryCfgMgr.logger.Info("conflict in updating canary config", + zap.Error(err), + zap.String("name", cfgName), + zap.String("namespace", cfgNamespace)) continue default: - log.Printf("Error updating Canary Config : %s.%s = %v", cfgName, cfgNamespace, err) - return err + e := "error updating canary config" + canaryCfgMgr.logger.Error(e, + zap.Error(err), + zap.String("name", cfgName), + zap.String("namespace", cfgNamespace)) + return errors.Wrapf(err, "%s: %s.%s", e, cfgName, cfgNamespace) } } @@ -358,7 +461,7 @@ func (canaryCfgMgr *canaryConfigMgr) rollForward(canaryConfig *crd.CanaryConfig, } } - log.Printf("Incremented functionWeights : %v", functionWeights) + canaryCfgMgr.logger.Info("incremented functionWeights", zap.Any("function_weights", functionWeights)) err := canaryCfgMgr.updateHttpTriggerWithRetries(trigger.Metadata.Name, trigger.Metadata.Namespace, functionWeights) return doneProcessingCanaryConfig, err @@ -369,7 +472,10 @@ func (canaryCfgMgr *canaryConfigMgr) reSyncCanaryConfigs() { canaryConfig := obj.(*crd.CanaryConfig) _, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata) if err != nil && canaryConfig.Status.Status == fission.CanaryConfigStatusPending { - log.Printf("Adding canary config : %s.%s from resync loop", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace) + canaryCfgMgr.logger.Info("adding canary config from resync loop", + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) // new canaryConfig detected, add it to our cache and start processing it go canaryCfgMgr.addCanaryConfig(canaryConfig) @@ -378,10 +484,17 @@ func (canaryCfgMgr *canaryConfigMgr) reSyncCanaryConfigs() { } func (canaryCfgMgr *canaryConfigMgr) deleteCanaryConfig(canaryConfig *crd.CanaryConfig) { - log.Printf("Delete event received for canary config : %v, %v, %v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, canaryConfig.Metadata.ResourceVersion) + canaryCfgMgr.logger.Info("delete event received for canary config", + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) canaryProcessingInfo, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata) if err != nil { - log.Printf("lookup of canaryConfig failed, err : %v", err) + canaryCfgMgr.logger.Error("lookup of canary config for deletion failed", + zap.Error(err), + zap.String("name", canaryConfig.Metadata.Name), + zap.String("namespace", canaryConfig.Metadata.Namespace), + zap.String("version", canaryConfig.Metadata.ResourceVersion)) return } // first stop the ticker @@ -396,7 +509,11 @@ func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfig(oldCanaryConfig *crd.Can err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&oldCanaryConfig.Metadata) if err != nil { - log.Printf("error removing canary config: %s from map, err : %v", oldCanaryConfig.Metadata.Name, err) + canaryCfgMgr.logger.Error("error removing canary config from map", + zap.Error(err), + zap.String("name", oldCanaryConfig.Metadata.Name), + zap.String("namespace", oldCanaryConfig.Metadata.Namespace), + zap.String("version", oldCanaryConfig.Metadata.ResourceVersion)) return } canaryCfgMgr.addCanaryConfig(newCanaryConfig) diff --git a/canaryconfigmgr/prometheusClient.go b/canaryconfigmgr/prometheusClient.go index 4ab1b075..8d3f9895 100644 --- a/canaryconfigmgr/prometheusClient.go +++ b/canaryconfigmgr/prometheusClient.go @@ -20,26 +20,26 @@ import ( "fmt" "time" - "golang.org/x/net/context" - + "github.com/pkg/errors" promClient "github.com/prometheus/client_golang/api/prometheus" "github.com/prometheus/common/model" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" + "golang.org/x/net/context" ) type PrometheusApiClient struct { + logger *zap.Logger client promClient.QueryAPI } -func MakePrometheusClient(prometheusSvc string) (*PrometheusApiClient, error) { +func MakePrometheusClient(logger *zap.Logger, prometheusSvc string) (*PrometheusApiClient, error) { promApiConfig := promClient.Config{ Address: prometheusSvc, } promApiClient, err := promClient.New(promApiConfig) if err != nil { - log.Errorf("Error creating prometheus api client for svc : %s, err : %v", prometheusSvc, err) - return nil, err + return nil, errors.Wrapf(err, "error creating prometheus api client for svc: %s", prometheusSvc) } apiQueryClient := promClient.NewQueryAPI(promApiClient) @@ -56,12 +56,12 @@ func MakePrometheusClient(prometheusSvc string) (*PrometheusApiClient, error) { } if err != nil { - log.Printf("Error sending test query to prometheus server: %v", err) - return nil, err + return nil, errors.Wrap(err, "error sending test query to prometheus server") } - log.Printf("Successfully made prometheus client with service : %s", prometheusSvc) + logger.Info("successfully made prometheus client with service", zap.String("service", prometheusSvc)) return &PrometheusApiClient{ + logger: logger.Named("prometheus_api_client"), client: apiQueryClient, }, nil } @@ -74,7 +74,7 @@ func (promApiClient *PrometheusApiClient) GetFunctionFailurePercentage(path, met } if reqs <= 0 { - return -1, fmt.Errorf("no requests to this url %v and method %v in the window : %v", path, method, window) + return -1, fmt.Errorf("no requests to this url %v and method %v in the window: %v", path, method, window) } // next, get a total count of errored out requests to this function in the same window @@ -94,20 +94,22 @@ func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string, reqs, err := promApiClient.executeQuery(queryString) if err != nil { - log.Printf("Error executing query : %s, err : %v", queryString, err) - return 0, err + return 0, errors.Wrapf(err, "error executing query: %s", queryString) } queryString = fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"} offset %v", path, method, funcName, funcNs, window) reqsInPrevWindow, err := promApiClient.executeQuery(queryString) if err != nil { - log.Printf("Error executing query : %s, err : %v", queryString, err) - return 0, err + return 0, errors.Wrapf(err, "error executing query: %s", queryString) } reqsInCurrentWindow := reqs - reqsInPrevWindow - log.Printf("reqs : %v, reqsInPrevWindow : %v, reqsInCurrentWindow : %v to function %v", reqs, reqsInPrevWindow, reqsInCurrentWindow, funcName) + promApiClient.logger.Info("function requests", + zap.Float64("requests", reqs), + zap.Float64("requests_in_previous_window", reqsInPrevWindow), + zap.Float64("requests_in_current_window", reqsInCurrentWindow), + zap.String("function", funcName)) return reqsInCurrentWindow, nil } @@ -117,20 +119,22 @@ func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(f failedRequests, err := promApiClient.executeQuery(queryString) if err != nil { - log.Printf("Error executing query : %s, err : %v", queryString, err) - return 0, err + return 0, errors.Wrapf(err, "error executing query: %s", queryString) } queryString = fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"} offset %v", funcName, funcNs, path, method, window) failedReqsInPrevWindow, err := promApiClient.executeQuery(queryString) if err != nil { - log.Printf("Error executing query : %s, err : %v", queryString, err) - return 0, err + return 0, errors.Wrapf(err, "error executing query: %s", queryString) } failedReqsInCurrentWindow := failedRequests - failedReqsInPrevWindow - log.Printf("failedReqs : %v, failedReqsInPrevWindow : %v, failedReqsInCurrentWindow : %v to function : %v", failedRequests, failedReqsInPrevWindow, failedReqsInCurrentWindow, funcName) + promApiClient.logger.Info("function requests", + zap.Float64("failed_requests", failedRequests), + zap.Float64("failed_requests_in_previous_window", failedReqsInPrevWindow), + zap.Float64("failed_requests_in_current_window", failedReqsInCurrentWindow), + zap.String("function", funcName)) return failedReqsInCurrentWindow, nil } @@ -138,8 +142,7 @@ func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(f func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (float64, error) { val, err := promApiClient.client.Query(context.Background(), queryString, time.Now()) if err != nil { - log.Errorf("Error querying prometheus qs : %v, err : %v", queryString, err) - return 0, err + return 0, errors.Wrapf(err, "error querying prometheus") } switch { @@ -159,23 +162,13 @@ func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (floa matrixVal := val.(model.Matrix) total := float64(0) for _, elem := range matrixVal { - //log.Printf("Only one value, so taking the 0th elem") total += float64(elem.Values[len(elem.Values)-1].Value) } return total, nil default: - log.Printf("type unrecognized") + promApiClient.logger.Info("return value type of prometheus query was unrecognized", + zap.Any("type", val.Type())) return 0, nil } } - -func addInterval(window string) string { - timeDuration, _ := time.ParseDuration(window) - log.Println("window in seconds", int64(timeDuration/time.Second)) - - timeInStr := fmt.Sprintf("%ds", int64((timeDuration+timeDuration)/time.Second)) - fmt.Println(timeInStr) - - return timeInStr -} diff --git a/charts/fission-all/templates/fluentd.yaml b/charts/fission-all/templates/fluentd.yaml index 34b2bbb1..1eec4dc7 100644 --- a/charts/fission-all/templates/fluentd.yaml +++ b/charts/fission-all/templates/fluentd.yaml @@ -28,7 +28,7 @@ spec: spec: containers: - name: fluentd - image: "{{ .Values.repository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}" + image: "{{ .Values.logger.fluentdImageRepository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}" imagePullPolicy: {{ .Values.pullPolicy }} env: - name: INFLUXDB_ADDRESS diff --git a/charts/fission-all/values.yaml b/charts/fission-all/values.yaml index c476152c..38ba379d 100644 --- a/charts/fission-all/values.yaml +++ b/charts/fission-all/values.yaml @@ -53,6 +53,7 @@ enableIstio: false ## Logger config logger: influxdbAdmin: "admin" + fluentdImageRepository: index.docker.io fluentdImage: fission/fluentd fluentdImageTag: 1.0.0 diff --git a/common.go b/common.go index 2cb97935..5ad55240 100644 --- a/common.go +++ b/common.go @@ -18,6 +18,7 @@ package fission import ( "fmt" + "io" "io/ioutil" "net" "net/http" @@ -28,6 +29,8 @@ import ( "strings" "syscall" + "go.uber.org/zap" + "github.com/gorilla/handlers" "github.com/imdario/mergo" "github.com/mholt/archiver" @@ -67,14 +70,31 @@ func GetFunctionIstioServiceName(fnName, fnNamespace string) string { return fmt.Sprintf("istio-%v-%v", fnName, fnNamespace) } -func LoggingMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestURI := r.RequestURI - if !strings.Contains(requestURI, "healthz") { - // Call the next handler, which can be another middleware in the chain, or the final handler. - handlers.LoggingHandler(os.Stdout, next).ServeHTTP(w, r) - } - }) +func LoggingMiddleware(logger *zap.Logger) func(next http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestURI := r.RequestURI + if !strings.Contains(requestURI, "healthz") { + // Call the next handler, which can be another middleware in the chain, or the final handler. + handlers.CustomLoggingHandler(os.Stdout, next, func(writer io.Writer, params handlers.LogFormatterParams) { + host, _, err := net.SplitHostPort(params.Request.RemoteAddr) + + if err != nil { + host = params.Request.RemoteAddr + } + + logger.Info("handled", + zap.String("host", host), + zap.String("method", params.Request.Method), + zap.String("uri", params.Request.RequestURI), + zap.String("proto", params.Request.Proto), + zap.Int("status_code", params.StatusCode), + zap.Int("size", params.Size)) + + }).ServeHTTP(w, r) + } + }) + } } // MergeContainerSpecs merges container specs using a predefined order. diff --git a/commonrbacutil.go b/commonrbacutil.go index e2604e04..18c8971b 100644 --- a/commonrbacutil.go +++ b/commonrbacutil.go @@ -18,8 +18,10 @@ package fission import ( "fmt" - "log" + "go.uber.org/zap" + + "github.com/pkg/errors" apiv1 "k8s.io/api/core/v1" rbac "k8s.io/api/rbac/v1beta1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -101,7 +103,7 @@ type PatchSpec struct { } // AddSaToRoleBindingWithRetries adds a service account to a rolebinding object. IT retries on already exists and conflict errors. -func AddSaToRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind string) (err error) { +func AddSaToRoleBindingWithRetries(logger *zap.Logger, k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind string) (err error) { patch := PatchSpec{} patch.Op = "add" patch.Path = "/subjects/-" @@ -113,34 +115,44 @@ func AddSaToRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding, patchJson, err := json.Marshal([]PatchSpec{patch}) if err != nil { - log.Printf("Error marshalling patch into json") + logger.Error("error marshalling patch into json", zap.Error(err)) return err } for i := 0; i < MaxRetries; i++ { _, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Patch(roleBinding, types.JSONPatchType, patchJson) if err == nil { - log.Printf("Patched rolebinding : %s.%s", roleBinding, roleBindingNs) + logger.Info("patched rolebinding", + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) return err } if k8serrors.IsNotFound(err) { + logger.Info("rolebinding not found - will try to create it", + zap.Error(err), + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) // someone may have deleted the object between us checking if the object is present and deciding to patch // so just create the object again rbObj := makeRoleBindingObj(roleBinding, roleBindingNs, role, roleKind, sa, saNamespace) rbObj, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Create(rbObj) if err == nil { - log.Printf("Created rolebinding : %s.%s", roleBinding, roleBindingNs) + logger.Info("created rolebinding", + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) return err } if k8serrors.IsAlreadyExists(err) { - log.Printf("rolebinding object : %s.%s already exists, retrying patch, err: %v", roleBinding, roleBindingNs, err) + logger.Info("rolebinding object already exists, retrying patch", + zap.Error(err), + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) continue } - log.Printf("Error returned by rolebinding create, %v", err) - return err + return errors.Wrap(err, "error returned by rolebinding create") } if k8serrors.IsConflict(err) { @@ -149,27 +161,30 @@ func AddSaToRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding, // but one CI run did show patch errored out on conflict : https://api.travis-ci.org/v3/job/373161490/log.txt, look for : // Error returned by rolebinding patch : there is a meaningful conflict (firstResourceVersion: "35482724", currentResourceVersion: "35482849") // so, m guessing retrying patch should help. will watch out for any such conflicts and fix the issue if any - log.Printf("Conflict reported on patch of rolebinding : %s.%s. Retrying patch operation, err : %v", roleBinding, roleBindingNs, err) + logger.Info("conflict reported on patch of rolebinding - retrying patch operation", + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) continue } - log.Printf("Error returned by rolebinding patch : %v", err) - return err + return errors.Wrap(err, "error returned by rolebinding patch") } - log.Printf("Exceeded maxretries : %d adding SA: %s.%s to rolebinding : %s.%s, giving up, err: %v", MaxRetries, sa, saNamespace, roleBinding, roleBindingNs, err) - return err + return errors.Wrapf(err, "exceeded max retries (%d) adding SA: %s.%s to rolebinding: %s.%s, giving up", MaxRetries, sa, saNamespace, roleBinding, roleBindingNs) } // RemoveSAFromRoleBindingWithRetries removes an SA from the rolebinding passed as parameter. If this is the only SA in // the rolebinding, then it deletes the rolebinding object. -func RemoveSAFromRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs string, saToRemove map[string]bool) (err error) { +func RemoveSAFromRoleBindingWithRetries(logger *zap.Logger, k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs string, saToRemove map[string]bool) (err error) { for i := 0; i < MaxRetries; i++ { rbObj, err := k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Get( roleBinding, metav1.GetOptions{}) if err != nil { // silently ignoring the error. there's no need for us to remove sa anymore. - log.Printf("rolebinding %s.%s not found, but ignoring the error since we're cleaning up", roleBinding, roleBindingNs) + logger.Info("rolebinding not found, but ignoring the error since we're cleaning up", + zap.Error(err), + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) return nil } @@ -198,43 +213,64 @@ func RemoveSAFromRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBin _, err = k8sClient.RbacV1beta1().RoleBindings(rbObj.Namespace).Update(rbObj) switch { case err == nil: - log.Printf("Removed sa's : %v from rolebinding : %s.%s", saToRemove, roleBinding, roleBindingNs) + logger.Info("removed service accounts from rolebinding", + zap.Any("service_accounts", saToRemove), + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) return nil case k8serrors.IsConflict(err): - log.Printf("Conflict in update of rolebinding: %s.%s, retrying", roleBinding, roleBindingNs) + logger.Info("conflict in update of rolebinding - retrying", + zap.Error(err), + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) continue default: - log.Printf("Rolebinding Update Errored out : %v", err) - return err + return errors.Wrap(err, "rolebinding update errored out") } } - log.Printf("Maxretries : %d exceeded for removing SA's : %v from rolebinding %s.%s, giving up, err: %v", MaxRetries, saToRemove, roleBinding, roleBindingNs, err) - return err + return errors.Wrapf(err, "max retries: %d exceeded for removing SA's: %v from rolebinding %s.%s, giving up", MaxRetries, saToRemove, roleBinding, roleBindingNs) } // SetupRoleBinding adds a role to a service account if the rolebinding object is already present in the namespace. // if not, it creates a rolebinding object granting the role to the SA in the namespace. -func SetupRoleBinding(k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, role, roleKind, sa, saNamespace string) error { +func SetupRoleBinding(logger *zap.Logger, k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, role, roleKind, sa, saNamespace string) error { // get the role binding object rbObj, err := k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Get( roleBinding, metav1.GetOptions{}) if err == nil { if !isSAInRoleBinding(rbObj, sa, saNamespace) { - return AddSaToRoleBindingWithRetries(k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind) + logger.Info("service account is not present in the rolebinding - will add", + zap.String("service_account_name", sa), + zap.String("service_account_namespace", saNamespace), + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) + return AddSaToRoleBindingWithRetries(logger, k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind) } - log.Printf("SA : %s.%s already present in rolebinding : %s.%s, so nothing to add", sa, saNamespace, roleBinding, roleBindingNs) + logger.Info("service account already present in rolebinding so nothing to add", + zap.String("service_account_name", sa), + zap.String("service_account_namespace", saNamespace), + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) return nil } // if role binding is missing, create it. also add this sa to the binding. if k8serrors.IsNotFound(err) { - log.Printf("Rolebinding %s does NOT exist in ns %s. Creating it", roleBinding, roleBindingNs) + logger.Info("rolebinding does NOT exist in namespace - creating it", + zap.Error(err), + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) rbObj = makeRoleBindingObj(roleBinding, roleBindingNs, role, roleKind, sa, saNamespace) rbObj, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Create(rbObj) if k8serrors.IsAlreadyExists(err) { - err = AddSaToRoleBindingWithRetries(k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind) + logger.Info("rolebinding already exists in namespace - adding service account to rolebinding", + zap.String("service_account_name", sa), + zap.String("service_account_namespace", saNamespace), + zap.String("role_binding", roleBinding), + zap.String("role_binding_namespace", roleBindingNs)) + err = AddSaToRoleBindingWithRetries(logger, k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind) } } diff --git a/controller/api.go b/controller/api.go index ef8dddf8..39925a37 100644 --- a/controller/api.go +++ b/controller/api.go @@ -24,7 +24,7 @@ import ( "strings" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,6 +46,7 @@ func init() { type ( API struct { + logger *zap.Logger fissionClient *crd.FissionClient kubernetesClient *kubernetes.Clientset storageServiceUrl string @@ -63,8 +64,8 @@ type ( } ) -func MakeAPI(featureStatus map[string]string) (*API, error) { - api, err := makeCRDBackedAPI() +func MakeAPI(logger *zap.Logger, featureStatus map[string]string) (*API, error) { + api, err := makeCRDBackedAPI(logger) u := os.Getenv("STORAGE_SERVICE_URL") if len(u) > 0 { @@ -119,7 +120,7 @@ func (api *API) respondWithError(w http.ResponseWriter, err error) { } code, msg := fission.GetHTTPError(err) - log.Errorf("Error: %v: %v", code, msg) + api.logger.Error(msg, zap.Int("code", code)) http.Error(w, msg, code) } @@ -270,7 +271,8 @@ func (api *API) Serve(port int) { address := fmt.Sprintf(":%v", port) - log.WithFields(log.Fields{"port": port}).Info("Server started") - r.Use(fission.LoggingMiddleware) - log.Fatal(http.ListenAndServe(address, r)) + api.logger.Info("server started", zap.Int("port", port)) + r.Use(fission.LoggingMiddleware(api.logger)) + err := http.ListenAndServe(address, r) + api.logger.Fatal("done listening", zap.Error(err)) } diff --git a/controller/api_test.go b/controller/api_test.go index c10a0cd4..d84511a0 100644 --- a/controller/api_test.go +++ b/controller/api_test.go @@ -28,6 +28,7 @@ import ( "testing" "time" + "go.uber.org/zap" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -344,7 +345,10 @@ func TestMain(m *testing.M) { return } - go Start(8888, true) + logger, err := zap.NewDevelopment() + panicIf(err) + + go Start(logger, 8888, true) time.Sleep(5 * time.Second) g.client = client.MakeClient("http://localhost:8888") diff --git a/controller/canaryConfigApi.go b/controller/canaryConfigApi.go index ae240572..c70db9d0 100644 --- a/controller/canaryConfigApi.go +++ b/controller/canaryConfigApi.go @@ -23,7 +23,7 @@ import ( "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" @@ -47,7 +47,7 @@ func (a *API) CanaryConfigApiCreate(w http.ResponseWriter, r *http.Request) { var canaryCfg crd.CanaryConfig err = json.Unmarshal(body, &canaryCfg) if err != nil { - log.Printf("Failed to unmarshal request body: [%v]", body) + a.logger.Error("failed to unmarshal request body", zap.Error(err), zap.Binary("body", body)) a.respondWithError(w, err) return } diff --git a/controller/config.go b/controller/config.go index c64f40fa..13c035dd 100644 --- a/controller/config.go +++ b/controller/config.go @@ -18,9 +18,9 @@ package controller import ( "context" - "fmt" - log "github.com/sirupsen/logrus" + "github.com/pkg/errors" + "go.uber.org/zap" "k8s.io/client-go/kubernetes" "github.com/fission/fission/canaryconfigmgr" @@ -28,24 +28,24 @@ import ( config "github.com/fission/fission/featureconfig" ) -func ConfigCanaryFeature(context context.Context, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, featureConfig *config.FeatureConfig, featureStatus map[string]string) error { +func ConfigCanaryFeature(context context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, featureConfig *config.FeatureConfig, featureStatus map[string]string) error { // start the appropriate controller if featureConfig.CanaryConfig.IsEnabled { - canaryCfgMgr, err := canaryconfigmgr.MakeCanaryConfigMgr(fissionClient, kubeClient, fissionClient.GetCrdClient(), + canaryCfgMgr, err := canaryconfigmgr.MakeCanaryConfigMgr(logger, fissionClient, kubeClient, fissionClient.GetCrdClient(), featureConfig.CanaryConfig.PrometheusSvc) if err != nil { featureStatus[config.CanaryFeature] = err.Error() - return fmt.Errorf("failed to start canary config manager: %v", err) + return errors.Wrap(err, "failed to start canary config manager") } canaryCfgMgr.Run(context) - log.Printf("Started canary config manager") + logger.Info("started canary config manager") } return nil } // ConfigureFeatures gets the feature config and configures the features that are enabled -func ConfigureFeatures(context context.Context, unitTestMode bool, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset) (map[string]string, error) { +func ConfigureFeatures(context context.Context, logger *zap.Logger, unitTestMode bool, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset) (map[string]string, error) { // set feature enabled to false if unitTestMode if unitTestMode { return nil, nil @@ -54,7 +54,7 @@ func ConfigureFeatures(context context.Context, unitTestMode bool, fissionClient // get the featureConfig from config map mounted onto the file system featureConfig, err := config.GetFeatureConfig() if err != nil { - log.Printf("Error getting feature config : %v", err) + logger.Error("error getting feature config", zap.Error(err)) return nil, err } @@ -62,6 +62,6 @@ func ConfigureFeatures(context context.Context, unitTestMode bool, fissionClient // configure respective features // in the future when new optional features are added, we need to add corresponding feature handlers and invoke them here - err = ConfigCanaryFeature(context, fissionClient, kubeClient, featureConfig, featureStatus) + err = ConfigCanaryFeature(context, logger, fissionClient, kubeClient, featureConfig, featureStatus) return featureStatus, err } diff --git a/controller/configmapApi.go b/controller/configmapApi.go index 6b6741df..7896eb06 100644 --- a/controller/configmapApi.go +++ b/controller/configmapApi.go @@ -21,7 +21,7 @@ import ( "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -35,7 +35,7 @@ func (a *API) ConfigMapGet(w http.ResponseWriter, r *http.Request) { configMap, err := a.kubernetesClient.CoreV1().ConfigMaps(ns).Get(name, metav1.GetOptions{}) if err != nil { - log.Printf("Error getting config map: %s from ns: %s", name, ns) + a.logger.Error("error getting config map", zap.Error(err), zap.String("config_map_name", name), zap.String("namespace", ns)) a.respondWithError(w, err) return } diff --git a/controller/controller.go b/controller/controller.go index f6a452bc..5a4d2895 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -18,41 +18,43 @@ package controller import ( "context" - "log" + + "go.uber.org/zap" "github.com/fission/fission" "github.com/fission/fission/crd" ) -func Start(port int, unitTestFlag bool) { +func Start(logger *zap.Logger, port int, unitTestFlag bool) { + cLogger := logger.Named("controller") // setup a signal handler for SIGTERM fission.SetupStackTraceHandler() fc, kc, apiExtClient, err := crd.MakeFissionClient() if err != nil { - log.Fatalf("Failed to connect to K8s API: %v", err) + cLogger.Fatal("failed to connect to k8s API", zap.Error(err)) } - err = crd.EnsureFissionCRDs(apiExtClient) + err = crd.EnsureFissionCRDs(cLogger, apiExtClient) if err != nil { - log.Fatalf("Failed to create fission CRDs: %v", err) + cLogger.Fatal("failed to create fission CRDs", zap.Error(err)) } err = fc.WaitForCRDs() if err != nil { - log.Fatalf("Error waiting for CRDs: %v", err) + cLogger.Fatal("error waiting for CRDs", zap.Error(err)) } ctx, cancel := context.WithCancel(context.Background()) - featureStatus, err := ConfigureFeatures(ctx, unitTestFlag, fc, kc) + featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc) if err != nil { - log.Printf("Error configuring features : %v. Proceeding without optional features", err.Error()) + cLogger.Info("error configuring features - proceeding without optional features", zap.Error(err)) } defer cancel() - api, err := MakeAPI(featureStatus) + api, err := MakeAPI(cLogger, featureStatus) if err != nil { - log.Fatalf("Failed to start controller: %v", err) + cLogger.Fatal("failed to start controller", zap.Error(err)) } api.Serve(port) } diff --git a/controller/crd.go b/controller/crd.go index 90fbd58b..61cf6202 100644 --- a/controller/crd.go +++ b/controller/crd.go @@ -17,13 +17,19 @@ limitations under the License. package controller import ( + "go.uber.org/zap" + "github.com/fission/fission/crd" ) -func makeCRDBackedAPI() (*API, error) { +func makeCRDBackedAPI(logger *zap.Logger) (*API, error) { fissionClient, kubernetesClient, _, err := crd.MakeFissionClient() if err != nil { return nil, err } - return &API{fissionClient: fissionClient, kubernetesClient: kubernetesClient}, nil + return &API{ + logger: logger.Named("api"), + fissionClient: fissionClient, + kubernetesClient: kubernetesClient, + }, nil } diff --git a/controller/environmentApi.go b/controller/environmentApi.go index febb3721..90000372 100644 --- a/controller/environmentApi.go +++ b/controller/environmentApi.go @@ -22,7 +22,7 @@ import ( "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" @@ -60,7 +60,7 @@ func (a *API) EnvironmentApiCreate(w http.ResponseWriter, r *http.Request) { var env crd.Environment err = json.Unmarshal(body, &env) if err != nil { - log.Printf("Failed to unmarshal request body: [%v]", body) + a.logger.Error("failed to unmarshal request body", zap.Error(err), zap.Binary("body", body)) a.respondWithError(w, err) return } diff --git a/controller/functionApi.go b/controller/functionApi.go index 05e64d0a..bdc00de2 100644 --- a/controller/functionApi.go +++ b/controller/functionApi.go @@ -27,7 +27,7 @@ import ( "sort" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" restclient "k8s.io/client-go/rest" @@ -188,7 +188,9 @@ func (a *API) FunctionLogsApiPost(w http.ResponseWriter, r *http.Request) { svcUrl, err := url.Parse(dbCnf.httpURL) if err != nil { - log.Printf("Failed to establish proxy server for function logs: %v", err) + a.logger.Error("failed parse url to establish proxy to database for function logs", + zap.Error(err), + zap.String("database_url", dbCnf.httpURL)) } // set up proxy server director director := func(req *http.Request) { diff --git a/controller/recordsApi.go b/controller/recordsApi.go index 40a6aff6..e801e486 100644 --- a/controller/recordsApi.go +++ b/controller/recordsApi.go @@ -26,7 +26,7 @@ import ( ) func (a *API) RecordsApiListAll(w http.ResponseWriter, r *http.Request) { - resp, err := redis.RecordsListAll() + resp, err := redis.RecordsListAll(a.logger.Named("redis")) if err != nil { a.respondWithError(w, err) return @@ -50,7 +50,7 @@ func (a *API) RecordsApiFilterByFunction(w http.ResponseWriter, r *http.Request) return } - resp, err := redis.RecordsFilterByFunction(query, recorders, triggers) + resp, err := redis.RecordsFilterByFunction(a.logger.Named("redis"), query, recorders, triggers) if err != nil { a.respondWithError(w, err) return @@ -74,7 +74,7 @@ func (a *API) RecordsApiFilterByTrigger(w http.ResponseWriter, r *http.Request) return } - resp, err := redis.RecordsFilterByTrigger(query, recorders, triggers) + resp, err := redis.RecordsFilterByTrigger(a.logger.Named("redis"), query, recorders, triggers) if err != nil { a.respondWithError(w, err) return @@ -86,7 +86,7 @@ func (a *API) RecordsApiFilterByTime(w http.ResponseWriter, r *http.Request) { from := r.FormValue("from") to := r.FormValue("to") - resp, err := redis.RecordsFilterByTime(from, to) + resp, err := redis.RecordsFilterByTime(a.logger.Named("redis"), from, to) if err != nil { a.respondWithError(w, err) return diff --git a/controller/replayAPI.go b/controller/replayAPI.go index 28e44b10..c1a7cc21 100644 --- a/controller/replayAPI.go +++ b/controller/replayAPI.go @@ -31,7 +31,7 @@ func (a *API) ReplayByReqUID(w http.ResponseWriter, r *http.Request) { routerUrl := fmt.Sprintf("http://router.%v", podNamespace) - resp, err := redis.ReplayByReqUID(routerUrl, queriedID) + resp, err := redis.ReplayByReqUID(a.logger, routerUrl, queriedID) if err != nil { a.respondWithError(w, err) return diff --git a/controller/secretApi.go b/controller/secretApi.go index 5e449da7..7a2cb91f 100644 --- a/controller/secretApi.go +++ b/controller/secretApi.go @@ -21,7 +21,7 @@ import ( "net/http" "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -35,7 +35,10 @@ func (a *API) SecretGet(w http.ResponseWriter, r *http.Request) { secret, err := a.kubernetesClient.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{}) if err != nil { - log.Printf("Error getting secret: %s from ns: %s", name, ns) + a.logger.Error("error getting secret", + zap.Error(err), + zap.String("secret_name", name), + zap.String("namespace", ns)) a.respondWithError(w, err) return } diff --git a/controller/storagesvc.go b/controller/storagesvc.go index 1b854d3f..647c8b06 100644 --- a/controller/storagesvc.go +++ b/controller/storagesvc.go @@ -18,19 +18,20 @@ package controller import ( "fmt" - "log" "net/http" "net/http/httputil" "net/url" + + "go.uber.org/zap" ) func (api *API) StorageServiceProxy(w http.ResponseWriter, r *http.Request) { u := api.storageServiceUrl ssUrl, err := url.Parse(u) if err != nil { - msg := fmt.Sprintf("Error parsing url %v: %v", u, err) - log.Println(msg) - http.Error(w, msg, http.StatusInternalServerError) + e := "error parsing url" + api.logger.Error(e, zap.Error(err), zap.String("url", u)) + http.Error(w, fmt.Sprintf("%s %s: %v", e, u, err), http.StatusInternalServerError) return } director := func(req *http.Request) { diff --git a/controller/workflowApiProxy.go b/controller/workflowApiProxy.go index ba8fbbdd..d99c18f8 100644 --- a/controller/workflowApiProxy.go +++ b/controller/workflowApiProxy.go @@ -2,10 +2,11 @@ package controller import ( "fmt" - "github.com/gorilla/mux" "net/http" "net/http/httputil" "net/url" + + "github.com/gorilla/mux" ) func (api *API) WorkflowApiserverProxy(w http.ResponseWriter, r *http.Request) { diff --git a/crd/crd.go b/crd/crd.go index 445c1a9d..99038c22 100644 --- a/crd/crd.go +++ b/crd/crd.go @@ -17,12 +17,12 @@ limitations under the License. package crd import ( - "log" "time" + "go.uber.org/zap" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -34,7 +34,7 @@ const ( // ensureCRD checks if the given CRD type exists, and creates it if // needed. (Note that this creates the CRD type; it doesn't create any // _instances_ of that type.) -func ensureCRD(clientset *apiextensionsclient.Clientset, crd *apiextensionsv1beta1.CustomResourceDefinition) (err error) { +func ensureCRD(logger *zap.Logger, clientset *apiextensionsclient.Clientset, crd *apiextensionsv1beta1.CustomResourceDefinition) (err error) { maxRetries := 5 for i := 0; i < maxRetries; i++ { @@ -44,12 +44,12 @@ func ensureCRD(clientset *apiextensionsclient.Clientset, crd *apiextensionsv1bet } // return if the resource already exists - if errors.IsAlreadyExists(err) { + if k8serrors.IsAlreadyExists(err) { return nil } else { // The requests fail to connect to k8s api server before // istio-prxoy is ready to serve traffic. Retry again. - log.Printf("Error connecting to kubernetes api service (%v), retrying", err) + logger.Info("error connecting to kubernetes api service, retrying", zap.Error(err)) time.Sleep(500 * time.Duration(2*i) * time.Millisecond) continue } @@ -59,7 +59,7 @@ func ensureCRD(clientset *apiextensionsclient.Clientset, crd *apiextensionsv1bet } // Ensure CRDs -func EnsureFissionCRDs(clientset *apiextensionsclient.Clientset) error { +func EnsureFissionCRDs(logger *zap.Logger, clientset *apiextensionsclient.Clientset) error { crds := []apiextensionsv1beta1.CustomResourceDefinition{ // Functions { @@ -207,7 +207,7 @@ func EnsureFissionCRDs(clientset *apiextensionsclient.Clientset) error { }, } for _, crd := range crds { - err := ensureCRD(clientset, &crd) + err := ensureCRD(logger, clientset, &crd) if err != nil { return err } diff --git a/crd/crd_test.go b/crd/crd_test.go index 16f109f0..01e86300 100644 --- a/crd/crd_test.go +++ b/crd/crd_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" @@ -427,8 +428,11 @@ func TestCrd(t *testing.T) { config, _, apiExtClient, err := GetKubernetesClient() panicIf(err) + logger, err := zap.NewDevelopment() + panicIf(err) + // init our types - err = EnsureFissionCRDs(apiExtClient) + err = EnsureFissionCRDs(logger, apiExtClient) panicIf(err) // rest client with knowledge about our crd types diff --git a/crd/recorder.go b/crd/recorder.go index 7e2a1405..434a54ae 100644 --- a/crd/recorder.go +++ b/crd/recorder.go @@ -19,7 +19,6 @@ package crd import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" ) diff --git a/environments/fetcher/client/client.go b/environments/fetcher/client/client.go index 81ff5985..bd591323 100644 --- a/environments/fetcher/client/client.go +++ b/environments/fetcher/client/client.go @@ -5,7 +5,9 @@ import ( "context" "encoding/json" "io/ioutil" - "log" + + "go.uber.org/zap" + "net/http" "strings" "time" @@ -18,14 +20,16 @@ import ( type ( Client struct { + logger *zap.Logger url string httpClient *http.Client } ) -func MakeClient(fetcherUrl string) *Client { +func MakeClient(logger *zap.Logger, fetcherUrl string) *Client { return &Client{ - url: strings.TrimSuffix(fetcherUrl, "/"), + logger: logger.Named("fetcher_client"), + url: strings.TrimSuffix(fetcherUrl, "/"), httpClient: &http.Client{ Transport: &ochttp.Transport{}, }, @@ -45,17 +49,17 @@ func (c *Client) getUploadUrl() string { } func (c *Client) Specialize(ctx context.Context, req *fission.FunctionSpecializeRequest) error { - _, err := sendRequest(ctx, c.httpClient, req, c.getSpecializeUrl()) + _, err := sendRequest(c.logger, ctx, c.httpClient, req, c.getSpecializeUrl()) return err } func (c *Client) Fetch(ctx context.Context, fr *fission.FunctionFetchRequest) error { - _, err := sendRequest(ctx, c.httpClient, fr, c.getFetchUrl()) + _, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getFetchUrl()) return err } func (c *Client) Upload(ctx context.Context, fr *fission.ArchiveUploadRequest) (*fission.ArchiveUploadResponse, error) { - body, err := sendRequest(ctx, c.httpClient, fr, c.getUploadUrl()) + body, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getUploadUrl()) uploadResp := fission.ArchiveUploadResponse{} err = json.Unmarshal(body, &uploadResp) @@ -66,7 +70,7 @@ func (c *Client) Upload(ctx context.Context, fr *fission.ArchiveUploadRequest) ( return &uploadResp, nil } -func sendRequest(ctx context.Context, httpClient *http.Client, req interface{}, url string) ([]byte, error) { +func sendRequest(logger *zap.Logger, ctx context.Context, httpClient *http.Client, req interface{}, url string) ([]byte, error) { body, err := json.Marshal(req) if err != nil { return nil, err @@ -82,7 +86,7 @@ func sendRequest(ctx context.Context, httpClient *http.Client, req interface{}, if resp.StatusCode == 200 { body, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Printf("Error reading response body: %v", err) + logger.Error("error reading response body", zap.Error(err)) } resp.Body.Close() return body, err @@ -92,7 +96,7 @@ func sendRequest(ctx context.Context, httpClient *http.Client, req interface{}, if i < maxRetries-1 { time.Sleep(50 * time.Duration(2*i) * time.Millisecond) - log.Printf("Error specializing/fetching/uploading package (%v) with url %v, retrying", err, url) + logger.Error("error specializing/fetching/uploading package, retrying", zap.Error(err), zap.String("url", url)) continue } } diff --git a/environments/fetcher/cmd/main.go b/environments/fetcher/cmd/main.go index cfdbc921..f3d8175d 100644 --- a/environments/fetcher/cmd/main.go +++ b/environments/fetcher/cmd/main.go @@ -12,6 +12,8 @@ import ( "runtime/debug" "syscall" + "go.uber.org/zap" + "go.opencensus.io/exporter/jaeger" "go.opencensus.io/plugin/ochttp" "go.opencensus.io/trace" @@ -49,14 +51,18 @@ func registerTraceExporter(collectorEndpoint string) error { // Usage: fetcher func main() { + logger, err := zap.NewProduction() + if err != nil { + log.Fatalf("can't initialize zap logger: %v", err) + } + defer logger.Sync() + // register signal handler for dumping stack trace. c := make(chan os.Signal, 1) signal.Notify(c, syscall.SIGTERM) go func() { <-c - log.Println("Received SIGTERM : Dumping stack trace") - dumpStackTrace() - os.Exit(1) + logger.Fatal("received SIGTERM") }() flag.Usage = fetcherUsage @@ -77,18 +83,18 @@ func main() { if os.IsNotExist(err) { err = os.MkdirAll(dir, os.ModeDir|0700) if err != nil { - log.Fatalf("Error creating directory: %v", err) + logger.Fatal("error creating directory", zap.Error(err), zap.String("directory", dir)) } } } if err := registerTraceExporter(*collectorEndpoint); err != nil { - log.Fatalf("Could not register trace exporter: %v", err) + logger.Fatal("could not register trace exporter", zap.Error(err), zap.String("collector_endpoint", *collectorEndpoint)) } - f, err := fetcher.MakeFetcher(dir, *secretDir, *configDir) + f, err := fetcher.MakeFetcher(logger, dir, *secretDir, *configDir) if err != nil { - log.Fatalf("Error making fetcher: %v", err) + logger.Fatal("error making fetcher", zap.Error(err)) } readyToServe := false @@ -100,13 +106,13 @@ func main() { err := json.Unmarshal([]byte(*specializePayload), &specializeReq) if err != nil { - log.Fatalf("Error decoding specialize request: %v", err) + logger.Fatal("error decoding specialize request", zap.Error(err)) } ctx := context.Background() err = f.SpecializePod(ctx, specializeReq.FetchReq, specializeReq.LoadReq) if err != nil { - log.Fatalf("Error specialing function poadt: %v", err) + logger.Fatal("error specializing function pod", zap.Error(err)) } readyToServe = true @@ -129,12 +135,12 @@ func main() { w.WriteHeader(http.StatusOK) }) - log.Println("Fetcher ready to receive requests") + logger.Info("fetcher ready to receive requests") http.ListenAndServe(":8000", &ochttp.Handler{ Handler: mux, }) } func fetcherUsage() { - fmt.Printf("Usage: fetcher [-specialize-on-startup] [-specialize-request ] [-secret-dir ] [-cfgmap-dir ] \n") + fmt.Println("Usage: fetcher [-specialize-on-startup] [-specialize-request ] [-secret-dir ] [-cfgmap-dir ] ") } diff --git a/environments/fetcher/fetcher.go b/environments/fetcher/fetcher.go index 7ef694d5..72c7643a 100644 --- a/environments/fetcher/fetcher.go +++ b/environments/fetcher/fetcher.go @@ -9,7 +9,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net" "net/http" "net/url" @@ -17,6 +16,8 @@ import ( "path/filepath" "time" + "go.uber.org/zap" + "github.com/mholt/archiver" "github.com/pkg/errors" uuid "github.com/satori/go.uuid" @@ -33,6 +34,7 @@ import ( type ( Fetcher struct { + logger *zap.Logger sharedVolumePath string sharedSecretPath string sharedConfigPath string @@ -42,23 +44,31 @@ type ( } ) -func makeVolumeDir(dirPath string) { - err := os.MkdirAll(dirPath, os.ModeDir|0700) - if err != nil { - log.Fatalf("Error creating %v: %v", dirPath, err) - } +func makeVolumeDir(dirPath string) error { + return os.MkdirAll(dirPath, os.ModeDir|0700) } -func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) { - makeVolumeDir(sharedVolumePath) - makeVolumeDir(sharedSecretPath) - makeVolumeDir(sharedConfigPath) +func MakeFetcher(logger *zap.Logger, sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) { + fLogger := logger.Named("fetcher") + err := makeVolumeDir(sharedVolumePath) + if err != nil { + fLogger.Fatal("error creating shared volume directory", zap.Error(err), zap.String("directory", sharedVolumePath)) + } + err = makeVolumeDir(sharedSecretPath) + if err != nil { + fLogger.Fatal("error creating shared secret directory", zap.Error(err), zap.String("directory", sharedSecretPath)) + } + err = makeVolumeDir(sharedConfigPath) + if err != nil { + fLogger.Fatal("error creating shared config directory", zap.Error(err), zap.String("directory", sharedConfigPath)) + } fissionClient, kubeClient, _, err := crd.MakeFissionClient() if err != nil { - return nil, err + return nil, errors.Wrap(err, "error making the fission / kube client") } return &Fetcher{ + logger: fLogger, sharedVolumePath: sharedVolumePath, sharedSecretPath: sharedSecretPath, sharedConfigPath: sharedConfigPath, @@ -138,9 +148,7 @@ func writeSecretOrConfigMap(dataMap map[string][]byte, dirPath string) error { writeFilePath := filepath.Join(dirPath, key) err := ioutil.WriteFile(writeFilePath, val, 0600) if err != nil { - e := fmt.Sprintf("Failed to write file %v: %v", writeFilePath, err) - log.Printf(e) - return errors.New(e) + return errors.Wrapf(err, "Failed to write file %s", writeFilePath) } } return nil @@ -160,38 +168,40 @@ func (fetcher *Fetcher) FetchHandler(w http.ResponseWriter, r *http.Request) { startTime := time.Now() defer func() { elapsed := time.Since(startTime) - log.Printf("elapsed time in fetch request = %v", elapsed) + fetcher.logger.Info("fetch request done", zap.Duration("elapsed_time", elapsed)) }() // parse request body, err := ioutil.ReadAll(r.Body) if err != nil { - log.Printf("Error reading request body") + fetcher.logger.Error("error reading request body", zap.Error(err)) http.Error(w, err.Error(), http.StatusInternalServerError) return } var req fission.FunctionFetchRequest err = json.Unmarshal(body, &req) if err != nil { - log.Printf("Error reading request body: %v", err) + fetcher.logger.Error("error parsing request body", zap.Error(err)) http.Error(w, err.Error(), http.StatusBadRequest) return } code, err := fetcher.Fetch(r.Context(), req) if err != nil { + fetcher.logger.Error("error fetching", zap.Error(err)) http.Error(w, err.Error(), code) return } - log.Printf("Checking secrets/cfgmaps") + fetcher.logger.Info("checking secrets/cfgmaps") code, err = fetcher.FetchSecretsAndCfgMaps(req.Secrets, req.ConfigMaps) if err != nil { + fetcher.logger.Error("error fetching secrets and config maps", zap.Error(err)) http.Error(w, err.Error(), code) return } - log.Printf("Completed fetch request") + fetcher.logger.Info("completed fetch request") // all done w.WriteHeader(http.StatusOK) } @@ -205,22 +215,21 @@ func (fetcher *Fetcher) SpecializeHandler(w http.ResponseWriter, r *http.Request // parse request body, err := ioutil.ReadAll(r.Body) if err != nil { - log.Printf("Error reading request body") + fetcher.logger.Error("error reading request body", zap.Error(err)) http.Error(w, err.Error(), http.StatusInternalServerError) return } var req fission.FunctionSpecializeRequest err = json.Unmarshal(body, &req) if err != nil { - log.Printf("Error reading request body: %v", err) + fetcher.logger.Error("error parsing request body", zap.Error(err)) http.Error(w, err.Error(), http.StatusBadRequest) return } - //log.Printf("fetcher received fetch request and started downloading: %v", req) - err = fetcher.SpecializePod(r.Context(), req.FetchReq, req.LoadReq) if err != nil { + fetcher.logger.Error("error specialing pod", zap.Error(err)) w.WriteHeader(http.StatusInternalServerError) return } @@ -234,14 +243,16 @@ func (fetcher *Fetcher) SpecializeHandler(w http.ResponseWriter, r *http.Request func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequest) (int, error) { // check that the requested filename is not an empty string and error out if so if len(req.Filename) == 0 { - e := fmt.Sprintf("Fetch request received for an empty file name, request: %v", req) - log.Printf(e) - return http.StatusBadRequest, errors.New(e) + e := "fetch request received for an empty file name" + fetcher.logger.Error(e, zap.Any("request", req)) + return http.StatusBadRequest, errors.New(fmt.Sprintf("%s, request: %v", e, req)) } // verify first if the file already exists. if _, err := os.Stat(filepath.Join(fetcher.sharedVolumePath, req.Filename)); err == nil { - log.Printf("Requested file: %s already exists at %s. Skipping fetch", req.Filename, fetcher.sharedVolumePath) + fetcher.logger.Info("requested file already exists at shared volume - skipping fetch", + zap.String("requested_file", req.Filename), + zap.String("shared_volume_path", fetcher.sharedVolumePath)) return http.StatusOK, nil } @@ -252,17 +263,19 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ // fetch the file and save it to the tmp path err := downloadUrl(ctx, fetcher.httpClient, req.Url, tmpPath) if err != nil { - e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err) - log.Printf(e) - return http.StatusBadRequest, errors.New(e) + e := "failed to download url" + fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url)) + return http.StatusBadRequest, errors.Wrapf(err, "%s: %s", e, req.Url) } } else { // get pkg pkg, err := fetcher.fissionClient.Packages(req.Package.Namespace).Get(req.Package.Name) if err != nil { - e := fmt.Sprintf("Failed to get package: %v", err) - log.Printf(e) - return http.StatusInternalServerError, errors.New(e) + e := "failed to get package" + fetcher.logger.Error(e, + zap.String("package_name", req.Package.Name), + zap.String("package_namespace", req.Package.Namespace)) + return http.StatusInternalServerError, errors.Wrap(err, e) } var archive *fission.Archive @@ -274,9 +287,12 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ // we hit this "Get : unsupported protocol scheme "" error. // it may be useful to the user if we can send a more meaningful error in such a scenario. if pkg.Status.BuildStatus != fission.BuildStatusSucceeded && pkg.Status.BuildStatus != fission.BuildStatusNone { - e := fmt.Sprintf("Build status for the function's pkg : %s.%s is : %s, can't fetch deployment", pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Status.BuildStatus) - log.Printf(e) - return http.StatusInternalServerError, errors.New(e) + e := fmt.Sprintf("cannot fetch deployment: package build status was not %q", fission.BuildStatusSucceeded) + fetcher.logger.Error(e, + zap.String("package_name", pkg.Metadata.Name), + zap.String("package_namespace", pkg.Metadata.Namespace), + zap.Any("package_build_status", pkg.Status.BuildStatus)) + return http.StatusInternalServerError, errors.New(fmt.Sprintf("%s: pkg %s.%s has a status of %s", e, pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Status.BuildStatus)) } archive = &pkg.Spec.Deployment } @@ -285,31 +301,31 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ // write pkg.Literal into tmpPath err = ioutil.WriteFile(tmpPath, archive.Literal, 0600) if err != nil { - e := fmt.Sprintf("Failed to write file %v: %v", tmpPath, err) - log.Printf(e) - return http.StatusInternalServerError, errors.New(e) + e := "failed to write file" + fetcher.logger.Error(e, zap.Error(err), zap.String("location", tmpPath)) + return http.StatusInternalServerError, errors.Wrapf(err, "%s %s", e, tmpPath) } } else { // download and verify err := downloadUrl(ctx, fetcher.httpClient, archive.URL, tmpPath) if err != nil { - e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err) - log.Printf(e) - return http.StatusBadRequest, errors.New(e) + e := "failed to download url" + fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url)) + return http.StatusBadRequest, errors.Wrapf(err, "%s %s", e, req.Url) } checksum, err := getChecksum(tmpPath) if err != nil { - e := fmt.Sprintf("Failed to get checksum: %v", err) - log.Printf(e) - return http.StatusBadRequest, errors.New(e) + e := "failed to get checksum" + fetcher.logger.Error(e, zap.Error(err)) + return http.StatusBadRequest, errors.Wrap(err, e) } err = verifyChecksum(checksum, &archive.Checksum) if err != nil { - e := fmt.Sprintf("Failed to verify checksum: %v", err) - log.Printf(e) - return http.StatusBadRequest, errors.New(e) + e := "failed to verify checksum" + fetcher.logger.Error(e, zap.Error(err)) + return http.StatusBadRequest, errors.Wrap(err, e) } } } @@ -319,7 +335,10 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ tmpUnarchivePath := filepath.Join(fetcher.sharedVolumePath, uuid.NewV4().String()) err := fetcher.unarchive(tmpPath, tmpUnarchivePath) if err != nil { - log.Println(err.Error()) + fetcher.logger.Error("error unarchiving", + zap.Error(err), + zap.String("archive_location", tmpPath), + zap.String("target_location", tmpUnarchivePath)) return http.StatusInternalServerError, err } @@ -327,13 +346,17 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ } // move tmp file to requested filename - err := fetcher.rename(tmpPath, filepath.Join(fetcher.sharedVolumePath, req.Filename)) + renamePath := filepath.Join(fetcher.sharedVolumePath, req.Filename) + err := fetcher.rename(tmpPath, renamePath) if err != nil { - log.Println(err.Error()) + fetcher.logger.Error("error renaming file", + zap.Error(err), + zap.String("original_path", tmpPath), + zap.String("rename_path", renamePath)) return http.StatusInternalServerError, err } - log.Printf("Successfully placed at %v", filepath.Join(fetcher.sharedVolumePath, req.Filename)) + fetcher.logger.Info("successfully placed", zap.String("location", renamePath)) return http.StatusOK, nil } @@ -345,13 +368,17 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference data, err := fetcher.kubeClient.CoreV1().Secrets(secret.Namespace).Get(secret.Name, metav1.GetOptions{}) if err != nil { - e := fmt.Sprintf("Failed to get secret from kubeapi: %v", err) - log.Printf(e) + e := "error getting secret from kubeapi" httpCode := http.StatusInternalServerError if k8serr.IsNotFound(err) { httpCode = http.StatusNotFound + e = "secret was not found in kubeapi" } + fetcher.logger.Error(e, + zap.Error(err), + zap.String("secret_name", secret.Name), + zap.String("secret_namespace", secret.Namespace)) return httpCode, errors.New(e) } @@ -360,12 +387,21 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference secretDir := filepath.Join(fetcher.sharedSecretPath, secretPath) err = os.MkdirAll(secretDir, os.ModeDir|0644) if err != nil { - e := fmt.Sprintf("Failed to create directory %v: %v", secretDir, err) - log.Printf(e) - return http.StatusInternalServerError, errors.New(e) + e := "failed to create directory for secret" + fetcher.logger.Error(e, + zap.Error(err), + zap.String("directory", secretDir), + zap.String("secret_name", secret.Name), + zap.String("secret_namespace", secret.Namespace)) + return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, secretDir) } err = writeSecretOrConfigMap(data.Data, secretDir) if err != nil { + fetcher.logger.Error("failed to write secret to file location", + zap.Error(err), + zap.String("location", secretDir), + zap.String("secret_name", secret.Name), + zap.String("secret_namespace", secret.Namespace)) return http.StatusInternalServerError, err } } @@ -376,13 +412,17 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference data, err := fetcher.kubeClient.CoreV1().ConfigMaps(config.Namespace).Get(config.Name, metav1.GetOptions{}) if err != nil { - e := fmt.Sprintf("Failed to get configmap from kubeapi: %v", err) - log.Printf(e) + e := "error getting configmap from kubeapi" httpCode := http.StatusInternalServerError if k8serr.IsNotFound(err) { httpCode = http.StatusNotFound + e = "configmap was not found in kubeapi" } + fetcher.logger.Error(e, + zap.Error(err), + zap.String("config_map_name", config.Name), + zap.String("config_map_namespace", config.Namespace)) return httpCode, errors.New(e) } @@ -391,9 +431,13 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference configDir := filepath.Join(fetcher.sharedConfigPath, configPath) err = os.MkdirAll(configDir, os.ModeDir|0644) if err != nil { - e := fmt.Sprintf("Failed to create directory %v: %v", configDir, err) - log.Printf(e) - return http.StatusInternalServerError, errors.New(e) + e := "failed to create directory for configmap" + fetcher.logger.Error(e, + zap.Error(err), + zap.String("directory", configDir), + zap.String("config_map_name", config.Name), + zap.String("config_map_namespace", config.Namespace)) + return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, configDir) } configMap := make(map[string][]byte) for key, val := range data.Data { @@ -401,6 +445,11 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference } err = writeSecretOrConfigMap(configMap, configDir) if err != nil { + fetcher.logger.Error("failed to write configmap to file location", + zap.Error(err), + zap.String("location", configDir), + zap.String("config_map_name", config.Name), + zap.String("config_map_namespace", config.Namespace)) return http.StatusInternalServerError, err } } @@ -418,13 +467,13 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) { startTime := time.Now() defer func() { elapsed := time.Since(startTime) - log.Printf("elapsed time in upload request = %v", elapsed) + fetcher.logger.Info("upload request done", zap.Duration("elapsed_time", elapsed)) }() // parse request body, err := ioutil.ReadAll(r.Body) if err != nil { - log.Printf("Error reading request body") + fetcher.logger.Error("error reading request body", zap.Error(err)) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -432,11 +481,11 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) { var req fission.ArchiveUploadRequest err = json.Unmarshal(body, &req) if err != nil { - log.Printf("Error reading request body: %v", err) + fetcher.logger.Error("error parsing request body", zap.Error(err)) http.Error(w, err.Error(), http.StatusBadRequest) return } - log.Printf("fetcher received upload request: %v", req) + fetcher.logger.Info("fetcher received upload request", zap.Any("request", req)) zipFilename := req.Filename + ".zip" srcFilepath := filepath.Join(fetcher.sharedVolumePath, req.Filename) @@ -445,37 +494,37 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) { if req.ArchivePackage { err = fetcher.archive(srcFilepath, dstFilepath) if err != nil { - e := fmt.Sprintf("Error archiving zip file: %v", err) - log.Println(e) - http.Error(w, e, http.StatusInternalServerError) + e := "error archiving zip file" + fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath)) + http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError) return } } else { err = os.Rename(srcFilepath, dstFilepath) if err != nil { - e := fmt.Sprintf("Error renaming the archive: %v", err) - log.Println(e) - http.Error(w, e, http.StatusInternalServerError) + e := "error renaming the archive" + fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath)) + http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError) return } } - log.Println("Starting upload...") + fetcher.logger.Info("starting upload...") ssClient := storageSvcClient.MakeClient(req.StorageSvcUrl) fileID, err := ssClient.Upload(r.Context(), dstFilepath, nil) if err != nil { - e := fmt.Sprintf("Error uploading zip file: %v", err) - log.Println(e) - http.Error(w, e, http.StatusInternalServerError) + e := "error uploading zip file" + fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath)) + http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError) return } sum, err := getChecksum(dstFilepath) if err != nil { - e := fmt.Sprintf("Error calculating checksum of zip file: %v", err) - log.Println(e) - http.Error(w, e, http.StatusInternalServerError) + e := "error calculating checksum of zip file" + fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath)) + http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError) return } @@ -486,13 +535,13 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) { rBody, err := json.Marshal(resp) if err != nil { - e := fmt.Sprintf("Error encoding upload response: %v", err) - log.Println(e) - http.Error(w, e, http.StatusInternalServerError) + e := "error encoding upload response" + fetcher.logger.Error(e, zap.Error(err)) + http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError) return } - log.Println("Completed upload request") + fetcher.logger.Info("completed upload request") w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) w.Write(rBody) @@ -501,7 +550,7 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) { func (fetcher *Fetcher) rename(src string, dst string) error { err := os.Rename(src, dst) if err != nil { - return errors.New(fmt.Sprintf("Failed to move file: %v", err)) + return errors.Wrap(err, "failed to move file") } return nil } @@ -512,7 +561,7 @@ func (fetcher *Fetcher) archive(src string, dst string) error { var files []string target, err := os.Stat(src) if err != nil { - return errors.New(fmt.Sprintf("Failed to zip file: %v", err)) + return errors.Wrap(err, "failed to zip file") } if target.IsDir() { // list all @@ -530,7 +579,7 @@ func (fetcher *Fetcher) archive(src string, dst string) error { func (fetcher *Fetcher) unarchive(src string, dst string) error { err := archiver.Zip.Open(src, dst) if err != nil { - return errors.New(fmt.Sprintf("Failed to unzip file: %v", err)) + return errors.Wrap(err, "failed to unzip file") } return nil } @@ -539,17 +588,17 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func startTime := time.Now() defer func() { elapsed := time.Since(startTime) - log.Printf("Elapsed time in fetch request = %v", elapsed) + fetcher.logger.Info("specialize request done", zap.Duration("elapsed_time", elapsed)) }() _, err := fetcher.Fetch(ctx, fetchReq) if err != nil { - return errors.Wrap(err, "Error fetching deploy package") + return errors.Wrap(err, "error fetching deploy package") } _, err = fetcher.FetchSecretsAndCfgMaps(fetchReq.Secrets, fetchReq.ConfigMaps) if err != nil { - return errors.Wrap(err, "Error fetching secrets/configmaps") + return errors.Wrap(err, "error fetching secrets/configmaps") } // Specialize the pod @@ -561,17 +610,19 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func loadPayload, err := json.Marshal(loadReq) if err != nil { - return errors.Wrap(err, "Error encoding load request") + return errors.Wrap(err, "error encoding load request") } if loadReq.EnvVersion >= 2 { contentType = "application/json" specializeURL = "http://localhost:8888/v2/specialize" reader = bytes.NewReader(loadPayload) + fetcher.logger.Info("calling environment v2 specialization endpoint") } else { contentType = "text/plain" specializeURL = "http://localhost:8888/specialize" reader = bytes.NewReader([]byte{}) + fetcher.logger.Info("calling environment v1 specialization endpoint") } for i := 0; i < maxRetries; i++ { @@ -588,7 +639,7 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func if netErr.Op == "dial" { if i < maxRetries-1 { time.Sleep(500 * time.Duration(2*i) * time.Millisecond) - log.Printf("Error connecting to pod (%v), retrying", netErr) + fetcher.logger.Error("error connecting to function environment pod for specialization request, retrying", zap.Error(netErr)) continue } } @@ -599,8 +650,8 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func err = fission.MakeErrorFromHTTP(resp) } - return errors.Wrap(err, "Error specializing function pod") + return errors.Wrap(err, "error specializing function pod") } - return errors.Wrap(err, fmt.Sprintf("Error specializing function pod after %v times", maxRetries)) + return errors.Wrapf(err, "error specializing function pod after %v times", maxRetries) } diff --git a/environments/go/server.go b/environments/go/server.go index 0e02456c..ceab1a5d 100644 --- a/environments/go/server.go +++ b/environments/go/server.go @@ -10,6 +10,7 @@ import ( "plugin" "github.com/pkg/errors" + "go.uber.org/zap" "github.com/fission/fission/environments/go/context" ) @@ -39,7 +40,7 @@ type ( var userFunc http.HandlerFunc -func loadPlugin(codePath, entrypoint string) (http.HandlerFunc, error) { +func loadPlugin(logger *zap.Logger, codePath, entrypoint string) (http.HandlerFunc, error) { // if codepath's a directory, load the file inside it info, err := os.Stat(codePath) @@ -58,7 +59,7 @@ func loadPlugin(codePath, entrypoint string) (http.HandlerFunc, error) { codePath = filepath.Join(codePath, fi.Name()) } - log.Printf("loading plugin from %v\n", codePath) + logger.Info("loading plugin", zap.String("location", codePath)) p, err := plugin.Open(codePath) if err != nil { return nil, errors.Wrap(err, "error loading plugin") @@ -85,87 +86,99 @@ func loadPlugin(codePath, entrypoint string) (http.HandlerFunc, error) { } } -func specializeHandler(w http.ResponseWriter, r *http.Request) { - if userFunc != nil { - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("Not a generic container")) - return - } - - _, err := os.Stat(CODE_PATH) - if err != nil { - if os.IsNotExist(err) { - w.WriteHeader(http.StatusNotFound) - log.Printf(err.Error()) - w.Write([]byte(CODE_PATH + ": not found")) - return - } else { - err = errors.Wrap(err, "unknown error") - log.Printf(err.Error()) - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) +func specializeHandler(logger *zap.Logger) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + if userFunc != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Not a generic container")) return } - } - log.Println("Specializing ...") - userFunc, err = loadPlugin(CODE_PATH, "Handler") - if err != nil { - err = errors.Wrap(err, "error specializing function") - log.Printf(err.Error()) - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - return + _, err := os.Stat(CODE_PATH) + if err != nil { + if os.IsNotExist(err) { + w.WriteHeader(http.StatusNotFound) + logger.Error("code path does not exist", + zap.Error(err), + zap.String("code_path", CODE_PATH)) + w.Write([]byte(CODE_PATH + ": not found")) + return + } else { + logger.Error("unknown error looking for code path", + zap.Error(err), + zap.String("code_path", CODE_PATH)) + err = errors.Wrap(err, "unknown error") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + } + + logger.Info("specializing ...") + userFunc, err = loadPlugin(logger, CODE_PATH, "Handler") + if err != nil { + e := "error specializing function" + logger.Error(e, zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(errors.Wrap(err, e).Error())) + return + } + logger.Info("done") } - log.Println("Done") } -func specializeHandlerV2(w http.ResponseWriter, r *http.Request) { - if userFunc != nil { - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("Not a generic container")) - return - } - - body, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Println(err.Error()) - w.WriteHeader(http.StatusInternalServerError) - return - } - var loadreq FunctionLoadRequest - err = json.Unmarshal(body, &loadreq) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - return - } - - _, err = os.Stat(loadreq.FilePath) - if err != nil { - if os.IsNotExist(err) { - w.WriteHeader(http.StatusNotFound) - log.Printf(err.Error()) - w.Write([]byte(CODE_PATH + ": not found")) - return - } else { - err = errors.Wrap(err, "unknown error") - log.Printf(err.Error()) - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) +func specializeHandlerV2(logger *zap.Logger) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + if userFunc != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Not a generic container")) return } - } - log.Println("Specializing ...") - userFunc, err = loadPlugin(loadreq.FilePath, loadreq.FunctionName) - if err != nil { - err = errors.Wrap(err, "error specializing function") - log.Printf(err.Error()) - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(err.Error())) - return + body, err := ioutil.ReadAll(r.Body) + if err != nil { + logger.Error("error reading request body", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + var loadreq FunctionLoadRequest + err = json.Unmarshal(body, &loadreq) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + _, err = os.Stat(loadreq.FilePath) + if err != nil { + if os.IsNotExist(err) { + logger.Error("code path does not exist", + zap.Error(err), + zap.String("code_path", CODE_PATH)) + w.WriteHeader(http.StatusNotFound) + w.Write([]byte(CODE_PATH + ": not found")) + return + } else { + logger.Error("unknown error looking for code path", + zap.Error(err), + zap.String("code_path", CODE_PATH)) + err = errors.Wrap(err, "unknown error") + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(err.Error())) + return + } + } + + logger.Info("specializing ...") + userFunc, err = loadPlugin(logger, loadreq.FilePath, loadreq.FunctionName) + if err != nil { + e := "error specializing function" + logger.Error(e, zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(errors.Wrap(err, e).Error())) + return + } + logger.Info("done") } - log.Println("Done") } func readinessProbeHandler(w http.ResponseWriter, r *http.Request) { @@ -173,9 +186,15 @@ func readinessProbeHandler(w http.ResponseWriter, r *http.Request) { } func main() { + logger, err := zap.NewProduction() + if err != nil { + log.Fatalf("can't initialize zap logger: %v", err) + } + defer logger.Sync() + http.HandleFunc("/healthz", readinessProbeHandler) - http.HandleFunc("/specialize", specializeHandler) - http.HandleFunc("/v2/specialize", specializeHandlerV2) + http.HandleFunc("/specialize", specializeHandler(logger.Named("specialize_handler"))) + http.HandleFunc("/v2/specialize", specializeHandlerV2(logger.Named("specialize_v2_handler"))) // Generic route -- all http requests go to the user function. http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -187,6 +206,6 @@ func main() { userFunc(w, r) }) - log.Println("Listening on 8888 ...") + logger.Info("listening on 8888 ...") http.ListenAndServe(":8888", nil) } diff --git a/executor/api.go b/executor/api.go index 3bf51223..835ee49f 100644 --- a/executor/api.go +++ b/executor/api.go @@ -21,12 +21,12 @@ import ( "encoding/json" "fmt" "io/ioutil" - "log" "net/http" "strings" "github.com/gorilla/mux" "go.opencensus.io/plugin/ochttp" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" @@ -50,7 +50,10 @@ func (executor *Executor) getServiceForFunctionApi(w http.ResponseWriter, r *htt serviceName, err := executor.getServiceForFunction(r.Context(), &m) if err != nil { code, msg := fission.GetHTTPError(err) - log.Printf("Error: %v: %v", code, msg) + executor.logger.Error("error getting service for function", + zap.Error(err), + zap.String("function", m.Name), + zap.String("fission_http_error", msg)) http.Error(w, msg, code) return } @@ -69,14 +72,19 @@ func (executor *Executor) getServiceForFunctionApi(w http.ResponseWriter, r *htt // invalidates the cache entry if the pod address was cached. func (executor *Executor) getServiceForFunction(ctx context.Context, m *metav1.ObjectMeta) (string, error) { // Check function -> svc cache - log.Printf("[%v] Checking for cached function service", m.Name) + executor.logger.Info("checking for cached function service", + zap.String("function_name", m.Name), + zap.String("function_namespace", m.Namespace)) fsvc, err := executor.fsCache.GetByFunction(m) if err == nil { if executor.isValidAddress(fsvc) { // Cached, return svc address return fsvc.Address, nil } else { - log.Printf("[%v] Deleting cache entry for invalid address : %s", m.Name, fsvc.Address) + executor.logger.Info("deleting cache entry for invalid address", + zap.String("function_name", m.Name), + zap.String("function_namespace", m.Namespace), + zap.String("address", fsvc.Address)) executor.fsCache.DeleteEntry(fsvc) } } @@ -98,6 +106,7 @@ func (executor *Executor) getServiceForFunction(ctx context.Context, m *metav1.O func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { + executor.logger.Error("failed to read tap service request", zap.Error(err)) http.Error(w, "Failed to read request", http.StatusInternalServerError) return } @@ -106,7 +115,10 @@ func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) { err = executor.fsCache.TouchByAddress(svcHost) if err != nil { - log.Printf("funcSvc tap error: %v", err) + executor.logger.Error("error tapping function service", + zap.Error(err), + zap.String("service", svcName), + zap.String("host", svcHost)) http.Error(w, "Not found", http.StatusNotFound) return } @@ -123,15 +135,15 @@ func (executor *Executor) Serve(port int) { r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST") r.HandleFunc("/healthz", executor.healthHandler).Methods("GET") address := fmt.Sprintf(":%v", port) - log.Printf("starting executor at port %v", port) + executor.logger.Info("starting executor", zap.Int("port", port)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() executor.ndm.Run(ctx) executor.gpm.Run(ctx) - r.Use(fission.LoggingMiddleware) + r.Use(fission.LoggingMiddleware(executor.logger)) err := http.ListenAndServe(address, &ochttp.Handler{ Handler: r, // Propagation: &b3.HTTPFormat{}, }) - log.Fatal(err) + executor.logger.Fatal("done listening", zap.Error(err)) } diff --git a/executor/client/client.go b/executor/client/client.go index 803cb958..ccd855e1 100644 --- a/executor/client/client.go +++ b/executor/client/client.go @@ -21,13 +21,14 @@ import ( "context" "encoding/json" "io/ioutil" - "log" "net/http" "net/url" "strings" "time" + "github.com/pkg/errors" "go.opencensus.io/plugin/ochttp" + "go.uber.org/zap" "golang.org/x/net/context/ctxhttp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -35,14 +36,16 @@ import ( ) type Client struct { + logger *zap.Logger executorUrl string tappedByUrl map[string]bool requestChan chan string httpClient *http.Client } -func MakeClient(executorUrl string) *Client { +func MakeClient(logger *zap.Logger, executorUrl string) *Client { c := &Client{ + logger: logger.Named("executor_client"), executorUrl: strings.TrimSuffix(executorUrl, "/"), tappedByUrl: make(map[string]bool), requestChan: make(chan string), @@ -59,12 +62,12 @@ func (c *Client) GetServiceForFunction(ctx context.Context, metadata *metav1.Obj body, err := json.Marshal(metadata) if err != nil { - return "", err + return "", errors.Wrap(err, "could not marshal request body for getting service for function") } resp, err := ctxhttp.Post(ctx, c.httpClient, executorUrl, "application/json", bytes.NewReader(body)) if err != nil { - return "", err + return "", errors.Wrap(err, "error posting to getting service for function") } defer resp.Body.Close() @@ -74,8 +77,7 @@ func (c *Client) GetServiceForFunction(ctx context.Context, metadata *metav1.Obj svcName, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Printf("Returning from ioutil read body") - return "", err + return "", errors.Wrap(err, "error reading response body from getting service for function") } return string(svcName), nil @@ -95,10 +97,10 @@ func (c *Client) service() { for u := range urls { err := c._tapService(u) if err != nil { - log.Printf("Error tapping function service address %v: %v", u, err) + c.logger.Error("error tapping function service address", zap.Error(err), zap.String("address", u)) } } - log.Printf("Tapped %v services in batch", len(urls)) + c.logger.Info("tapped services in batch", zap.Int("service_count", len(urls))) }() } } diff --git a/executor/executor.go b/executor/executor.go index 6ad7c95d..18bb40c2 100644 --- a/executor/executor.go +++ b/executor/executor.go @@ -19,7 +19,6 @@ package executor import ( "context" "fmt" - "log" "net/http" "runtime/debug" "strings" @@ -29,6 +28,7 @@ import ( "github.com/dchest/uniuri" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" @@ -41,6 +41,8 @@ import ( type ( Executor struct { + logger *zap.Logger + gpm *poolmgr.GenericPoolManager ndm *newdeploy.NewDeploy @@ -62,8 +64,9 @@ type ( } ) -func MakeExecutor(gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor { +func MakeExecutor(logger *zap.Logger, gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor { executor := &Executor{ + logger: logger.Named("executor"), gpm: gpm, ndm: ndm, fissionClient: fissionClient, @@ -111,7 +114,8 @@ func (executor *Executor) serveCreateFuncServices() { } else { // There's an existing request for this function, wait for it to finish go func() { - log.Printf("Waiting for concurrent request for the same function: %v", m) + executor.logger.Info("waiting for concurrent request for the same function", + zap.Any("function", m)) wg.Wait() // get the function service from the cache @@ -121,7 +125,9 @@ func (executor *Executor) serveCreateFuncServices() { // It normally happened if there are multiple requests are // waiting for the same function and executor failed to cre- // ate service for function. - err = errors.Wrap(err, fmt.Sprintf("Error getting service for function %v in namespace %v", m.Name, m.Namespace)) + err = errors.Wrapf(err, "error getting service for function", + zap.String("function_name", m.Name), + zap.String("function_namespace", m.Namespace)) req.respChan <- &createFuncServiceResponse{ funcSvc: fsvc, err: err, @@ -140,7 +146,9 @@ func (executor *Executor) getFunctionExecutorType(meta *metav1.ObjectMeta) (fiss } func (executor *Executor) createServiceForFunction(ctx context.Context, meta *metav1.ObjectMeta) (*fscache.FuncSvc, error) { - log.Printf("[%v] No cached function service found, creating one", meta.Name) + executor.logger.Info("no cached function service found, creating one", + zap.String("function_name", meta.Name), + zap.String("function_namespace", meta.Namespace)) executorType, err := executor.getFunctionExecutorType(meta) if err != nil { @@ -158,8 +166,12 @@ func (executor *Executor) createServiceForFunction(ctx context.Context, meta *me } if fsvcErr != nil { - fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%v] Error creating service for function", meta.Name)) - log.Print(fsvcErr) + e := "error creating service for function" + executor.logger.Error(e, + zap.Error(fsvcErr), + zap.String("function_name", meta.Name), + zap.String("function_namespace", meta.Namespace)) + fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%s] %s", meta.Name, e)) } else if fsvc != nil { _, err = executor.fsCache.Add(*fsvc) if err != nil { @@ -185,16 +197,18 @@ func dumpStackTrace() { debug.PrintStack() } -func serveMetric() { +func serveMetric(logger *zap.Logger) { // Expose the registered metrics via HTTP. metricAddr := ":8080" http.Handle("/metrics", promhttp.Handler()) - log.Fatal(http.ListenAndServe(metricAddr, nil)) + err := http.ListenAndServe(metricAddr, nil) + + logger.Fatal("done listening on metrics endpoint", zap.Error(err)) } // StartExecutor Starts executor and the executor components such as Poolmgr, // deploymgr and potential future executor types -func StartExecutor(fissionNamespace string, functionNamespace string, envBuilderNamespace string, port int) error { +func StartExecutor(logger *zap.Logger, fissionNamespace string, functionNamespace string, envBuilderNamespace string, port int) error { // setup a signal handler for SIGTERM fission.SetupStackTraceHandler() @@ -202,33 +216,34 @@ func StartExecutor(fissionNamespace string, functionNamespace string, envBuilder err = fissionClient.WaitForCRDs() if err != nil { - log.Fatalf("Error waiting for CRDs: %v", err) + return errors.Wrap(err, "error waiting for CRDs") } restClient := fissionClient.GetCrdClient() if err != nil { - log.Printf("Failed to get kubernetes client: %v", err) - return err + return errors.Wrap(err, "failed to get kubernetes client") } - fsCache := fscache.MakeFunctionServiceCache() + fsCache := fscache.MakeFunctionServiceCache(logger) poolID := strings.ToLower(uniuri.NewLen(8)) - reaper.CleanupOldExecutorObjects(kubernetesClient, poolID) - go reaper.CleanupRoleBindings(kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30) + reaper.CleanupOldExecutorObjects(logger, kubernetesClient, poolID) + go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30) gpm := poolmgr.MakeGenericPoolManager( + logger, fissionClient, kubernetesClient, functionNamespace, poolID) ndm := newdeploy.MakeNewDeploy( + logger, fissionClient, kubernetesClient, restClient, functionNamespace, poolID) - api := MakeExecutor(gpm, ndm, fissionClient, fsCache) + api := MakeExecutor(logger, gpm, ndm, fissionClient, fsCache) go api.Serve(port) - go serveMetric() + go serveMetric(logger) return nil } diff --git a/executor/executor_test.go b/executor/executor_test.go index c99ce473..e9dfe583 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -33,6 +33,7 @@ import ( "testing" "time" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -44,6 +45,12 @@ import ( "github.com/fission/fission/executor/client" ) +func panicIf(err error) { + if err != nil { + log.Panicf("Error: %v", err) + } +} + // return the number of pods in the given namespace matching the given labels func countPods(kubeClient *kubernetes.Clientset, ns string, labelz map[string]string) int { pods, err := kubeClient.Pods(ns).List(metav1.ListOptions{ @@ -134,8 +141,11 @@ func TestExecutor(t *testing.T) { createTestNamespace(kubeClient, functionNs) defer kubeClient.Namespaces().Delete(functionNs, nil) + logger, err := zap.NewDevelopment() + panicIf(err) + // make sure CRD types exist on cluster - err = crd.EnsureFissionCRDs(apiExtClient) + err = crd.EnsureFissionCRDs(logger, apiExtClient) if err != nil { log.Panicf("failed to ensure crds: %v", err) } @@ -165,13 +175,13 @@ func TestExecutor(t *testing.T) { // create poolmgr port := 9999 - err = StartExecutor(fissionNs, functionNs, port) + err = StartExecutor(logger, fissionNs, functionNs, "fission-builder", port) if err != nil { log.Panicf("failed to start poolmgr: %v", err) } // connect poolmgr client - poolmgrClient := client.MakeClient(fmt.Sprintf("http://localhost:%v", port)) + poolmgrClient := client.MakeClient(logger, fmt.Sprintf("http://localhost:%v", port)) // Wait for pool to be created (we don't actually need to do // this, since the API should do the right thing in any case). diff --git a/executor/fscache/functionServiceCache.go b/executor/fscache/functionServiceCache.go index 25acc3cb..5cd95b9b 100644 --- a/executor/fscache/functionServiceCache.go +++ b/executor/fscache/functionServiceCache.go @@ -17,9 +17,11 @@ limitations under the License. package fscache import ( - "log" + "fmt" "time" + "github.com/pkg/errors" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -57,6 +59,7 @@ type ( } FunctionServiceCache struct { + logger *zap.Logger byFunction *cache.Cache // function-key -> funcSvc : map[string]*funcSvc byAddress *cache.Cache // address -> function : map[string]metav1.ObjectMeta byFunctionUID *cache.Cache // function uid -> function : map[string]metav1.ObjectMeta @@ -91,8 +94,9 @@ func IsNameExistError(err error) bool { return false } -func MakeFunctionServiceCache() *FunctionServiceCache { +func MakeFunctionServiceCache(logger *zap.Logger) *FunctionServiceCache { fsc := &FunctionServiceCache{ + logger: logger.Named("function_service_cache"), byFunction: cache.MakeCache(0, 0), byAddress: cache.MakeCache(0, 0), byFunctionUID: cache.MakeCache(0, 0), @@ -122,14 +126,16 @@ func (fsc *FunctionServiceCache) service() { } resp.objects = funcObjects case LOG: + fsc.logger.Info("dumping function service cache") funcCopy := fsc.byFunction.Copy() - log.Printf("Cache has %v entries", len(funcCopy)) + info := []string{} for key, fsvcI := range funcCopy { fsvc := fsvcI.(*FuncSvc) for _, kubeObj := range fsvc.KubernetesObjects { - log.Printf("%v\t%v\t%v", key, kubeObj.Kind, kubeObj.Name) + info = append(info, fmt.Sprintf("%v\t%v\t%v", key, kubeObj.Kind, kubeObj.Name)) } } + fsc.logger.Info("function service cache", zap.Int("item_count", len(funcCopy)), zap.Strings("cache", info)) } req.responseChannel <- resp } @@ -196,9 +202,8 @@ func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) { if err != nil { if IsNameExistError(err) { err = nil - } - if err != nil { - log.Printf("error caching fsvc: %v", err) + } else { + err = errors.Wrap(err, "error caching fsvc") } return nil, err } @@ -209,9 +214,8 @@ func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) { if err != nil { if IsNameExistError(err) { err = nil - } - if err != nil { - log.Printf("error caching fsvc by function uid: %v", err) + } else { + err = errors.Wrap(err, "error caching fsvc by function uid") } return nil, err } @@ -278,12 +282,12 @@ func (fsc *FunctionServiceCache) ListOld(age time.Duration) ([]*FuncSvc, error) } func (fsc *FunctionServiceCache) Log() { - log.Printf("--- FunctionService Cache Contents") + fsc.logger.Info("--- FunctionService Cache Contents") responseChannel := make(chan *fscResponse) fsc.requestChannel <- &fscRequest{ requestType: LOG, responseChannel: responseChannel, } <-responseChannel - log.Printf("--- FunctionService Cache Contents End") + fsc.logger.Info("--- FunctionService Cache Contents End") } diff --git a/executor/fscache/functionServiceCache_test.go b/executor/fscache/functionServiceCache_test.go index f97ab834..38caa5ac 100644 --- a/executor/fscache/functionServiceCache_test.go +++ b/executor/fscache/functionServiceCache_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -12,8 +13,17 @@ import ( "github.com/fission/fission/crd" ) +func panicIf(err error) { + if err != nil { + log.Panicf("Error: %v", err) + } +} + func TestFunctionServiceCache(t *testing.T) { - fsc := MakeFunctionServiceCache() + logger, err := zap.NewDevelopment() + panicIf(err) + + fsc := MakeFunctionServiceCache(logger) if fsc == nil { log.Panicf("error creating cache") } diff --git a/executor/newdeploy/newdeploy.go b/executor/newdeploy/newdeploy.go index 18575524..423e95d4 100644 --- a/executor/newdeploy/newdeploy.go +++ b/executor/newdeploy/newdeploy.go @@ -20,11 +20,11 @@ import ( "encoding/json" "errors" "fmt" - "log" "path/filepath" "time" "github.com/hashicorp/go-multierror" + "go.uber.org/zap" asv1 "k8s.io/api/autoscaling/v1" apiv1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" @@ -59,10 +59,9 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Get(deployName, metav1.GetOptions{}) if err == nil { if waitForDeploy { - err = scaleDeployment(deploy.kubernetesClient, - existingDepl.Namespace, existingDepl.Name, minScale) + err = deploy.scaleDeployment(existingDepl.Namespace, existingDepl.Name, minScale) if err != nil { - log.Printf("Error scaling up deployment for function %v: %v", fn.Metadata.Name, err) + deploy.logger.Error("error scaling up function deployment", zap.Error(err), zap.String("function", fn.Metadata.Name)) return nil, err } @@ -86,7 +85,11 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro depl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Create(deployment) if err != nil { - log.Printf("Error while creating deployment: %v", err) + deploy.logger.Error("error while creating function deployment", + zap.Error(err), + zap.String("function", fn.Metadata.Name), + zap.String("deployment_name", deployName), + zap.String("deployment_namespace", deployNamespace)) return nil, err } @@ -105,25 +108,40 @@ func (deploy *NewDeploy) setupRBACObjs(deployNamespace string, fn *crd.Function) // create fetcher SA in this ns, if not already created _, err := fission.SetupSA(deploy.kubernetesClient, fission.FissionFetcherSA, deployNamespace) if err != nil { - log.Printf("Error : %v creating %s in ns : %s for function: %s.%s", err, fission.FissionFetcherSA, deployNamespace, fn.Metadata.Name, fn.Metadata.Namespace) + deploy.logger.Error("error creating fission fetcher service account for function", + zap.Error(err), + zap.String("service_account_name", fission.FissionFetcherSA), + zap.String("service_account_namespace", deployNamespace), + zap.String("function_name", fn.Metadata.Name), + zap.String("function_namespace", fn.Metadata.Namespace)) return err } // create a cluster role binding for the fetcher SA, if not already created, granting access to do a get on packages in any ns - err = fission.SetupRoleBinding(deploy.kubernetesClient, fission.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace) + err = fission.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, fission.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace) if err != nil { - log.Printf("Error : %v creating %s RoleBinding for function: %s.%s", err, fission.PackageGetterRB, fn.Metadata.Name, fn.Metadata.Namespace) + deploy.logger.Error("error creating role binding for function", + zap.Error(err), + zap.String("role_binding", fission.PackageGetterRB), + zap.String("function_name", fn.Metadata.Name), + zap.String("function_namespace", fn.Metadata.Namespace)) return err } // create rolebinding in function namespace for fetcherSA.envNamespace to be able to get secrets and configmaps - err = fission.SetupRoleBinding(deploy.kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace) + err = fission.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace) if err != nil { - log.Printf("Error : %v creating %s RoleBinding for function %s.%s", err, fission.SecretConfigMapGetterRB, fn.Metadata.Name, fn.Metadata.Namespace) + deploy.logger.Error("error creating role binding for function", + zap.Error(err), + zap.String("role_binding", fission.SecretConfigMapGetterRB), + zap.String("function_name", fn.Metadata.Name), + zap.String("function_namespace", fn.Metadata.Namespace)) return err } - log.Printf("Set up all RBAC objects for function : %s.%s", fn.Metadata.Name, fn.Metadata.Namespace) + deploy.logger.Info("set up all RBAC objects for function", + zap.String("function_name", fn.Metadata.Name), + zap.String("function_namespace", fn.Metadata.Namespace)) return nil } @@ -188,7 +206,7 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen fetcherResources, err := util.GetFetcherResources() if err != nil { - log.Printf("Error while parsing fetcher resources: %v", err) + deploy.logger.Error("error while parsing fetcher resources", zap.Error(err)) return nil, err } @@ -515,19 +533,28 @@ func (deploy *NewDeploy) cleanupNewdeploy(ns string, name string) error { err := deploy.deleteSvc(ns, name) if err != nil { - log.Printf("Error deleting service for newdeploy function %v in namespace %v, error: %v", name, ns, err) + deploy.logger.Error("error deleting service for newdeploy function", + zap.Error(err), + zap.String("function_name", name), + zap.String("function_namespace", ns)) multierror.Append(multierr, err) } err = deploy.deleteHpa(ns, name) if err != nil { - log.Printf("Error deleting HPA for newdeploy function %v in namespace %v, error: %v", name, ns, err) + deploy.logger.Error("error deleting service for newdeploy function", + zap.Error(err), + zap.String("function_name", name), + zap.String("function_namespace", ns)) multierror.Append(multierr, err) } err = deploy.deleteDeployment(ns, name) if err != nil { - log.Printf("Error deleting deployment for newdeploy function %v in namespace %v, error: %v", name, ns, err) + deploy.logger.Error("error deleting deployment for newdeploy function", + zap.Error(err), + zap.String("function_name", name), + zap.String("function_namespace", ns)) multierror.Append(multierr, err) } return multierr.ErrorOrNil() diff --git a/executor/newdeploy/newdeploymgr.go b/executor/newdeploy/newdeploymgr.go index 53187a76..460e3c7f 100644 --- a/executor/newdeploy/newdeploymgr.go +++ b/executor/newdeploy/newdeploymgr.go @@ -19,7 +19,6 @@ package newdeploy import ( "context" "fmt" - "log" "os" "strconv" "strings" @@ -29,6 +28,7 @@ import ( "github.com/fission/fission/throttler" multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" k8sErrs "k8s.io/apimachinery/pkg/api/errors" @@ -46,6 +46,8 @@ import ( type ( NewDeploy struct { + logger *zap.Logger + kubernetesClient *kubernetes.Clientset fissionClient *crd.FissionClient crdClient *rest.RESTClient @@ -72,6 +74,7 @@ type ( ) func MakeNewDeploy( + logger *zap.Logger, fissionClient *crd.FissionClient, kubernetesClient *kubernetes.Clientset, crdClient *rest.RESTClient, @@ -79,7 +82,7 @@ func MakeNewDeploy( instanceID string, ) *NewDeploy { - log.Printf("Creating NewDeploy ExecutorType") + logger.Info("creating NewDeploy ExecutorType") fetcherImg := os.Getenv("FETCHER_IMAGE") if len(fetcherImg) == 0 { @@ -92,19 +95,21 @@ func MakeNewDeploy( if len(os.Getenv("ENABLE_ISTIO")) > 0 { istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO")) if err != nil { - log.Println("Failed to parse ENABLE_ISTIO") + logger.Info("failed to parse 'ENABLE_ISTIO'") } enableIstio = istio } nd := &NewDeploy{ + logger: logger.Named("new_deploy"), + fissionClient: fissionClient, kubernetesClient: kubernetesClient, crdClient: crdClient, instanceID: instanceID, namespace: namespace, - fsCache: fscache.MakeFunctionServiceCache(), + fsCache: fscache.MakeFunctionServiceCache(logger), throttler: throttler.MakeThrottler(1 * time.Minute), fetcherImg: fetcherImg, @@ -142,14 +147,18 @@ func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controll fn := obj.(*crd.Function) _, err := deploy.createFunction(fn, true) if err != nil { - log.Printf("Error eager creating function: %v", err) + deploy.logger.Error("error eager creating function", + zap.Error(err), + zap.Any("function", fn)) } }, DeleteFunc: func(obj interface{}) { fn := obj.(*crd.Function) err := deploy.deleteFunction(fn) if err != nil { - log.Printf("Error deleing the function: %v", err) + deploy.logger.Error("error deleting function", + zap.Error(err), + zap.Any("function", fn)) } }, UpdateFunc: func(oldObj interface{}, newObj interface{}) { @@ -157,7 +166,10 @@ func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controll newFn := newObj.(*crd.Function) err := deploy.updateFunction(oldFn, newFn) if err != nil { - log.Printf("Error deleing the function: %v", err) + deploy.logger.Error("error updating function", + zap.Error(err), + zap.Any("old_function", oldFn), + zap.Any("new_function", newFn)) } }, }) @@ -186,7 +198,7 @@ func (deploy *NewDeploy) createFunction(fn *crd.Function, firstcreate bool) (*fs fsvc, ok := fsvcObj.(*fscache.FuncSvc) if !ok { - log.Panic("Receive unknown object, expect pointer of function service object") + deploy.logger.Panic("receive unknown object while creating function - expected pointer of function service object") } return fsvc, err @@ -198,7 +210,7 @@ func (deploy *NewDeploy) deleteFunction(fn *crd.Function) error { } err := deploy.fnDelete(fn) if err != nil { - err = errors.Wrap(err, fmt.Sprintf("error deleting kubernetes objects of function %v", fn.Metadata)) + err = errors.Wrapf(err, "error deleting kubernetes objects of function %v", fn.Metadata) } return err } @@ -235,22 +247,23 @@ func (deploy *NewDeploy) fnCreate(fn *crd.Function, firstcreate bool) (*fscache. // then deployment) to take advantage of waiting time. svc, err := deploy.createOrGetSvc(deployLabels, objName, ns) if err != nil { - log.Printf("Error creating the service %v: %v", objName, err) + deploy.logger.Error("error creating service", zap.Error(err), zap.String("service", objName)) go deploy.cleanupNewdeploy(ns, objName) - return nil, errors.Wrap(err, fmt.Sprintf("error creating service %v", objName)) + return nil, errors.Wrapf(err, "error creating service %v", objName) } svcAddress := fmt.Sprintf("%v.%v", svc.Name, svc.Namespace) depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels, ns, firstcreate) if err != nil { - log.Printf("Error creating the deployment %v: %v", objName, err) + deploy.logger.Error("error creating deployment", zap.Error(err), zap.String("deployment", objName)) go deploy.cleanupNewdeploy(ns, objName) - return nil, errors.Wrap(err, fmt.Sprintf("error creating deployment %v", objName)) + return nil, errors.Wrapf(err, "error creating deployment %v", objName) } hpa, err := deploy.createOrGetHpa(objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl) if err != nil { + deploy.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName)) go deploy.cleanupNewdeploy(ns, objName) - return nil, errors.Wrap(err, fmt.Sprintf("error creating the HPA %v:", objName)) + return nil, errors.Wrapf(err, "error creating the HPA %v", objName) } kubeObjRefs := []apiv1.ObjectReference{ @@ -292,7 +305,7 @@ func (deploy *NewDeploy) fnCreate(fn *crd.Function, firstcreate bool) (*fscache. _, err = deploy.fsCache.Add(*fsvc) if err != nil { - log.Printf("Error adding the function to cache: %v", err) + deploy.logger.Error("error adding function to cache", zap.Error(err), zap.Any("function", fsvc.Function)) return fsvc, err } return fsvc, nil @@ -313,7 +326,8 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function // Executor type is no longer New Deployment if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy && oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy { - log.Printf("function does not use new deployment executor anymore, deleting resources: %v", newFn) + deploy.logger.Info("function does not use new deployment executor anymore, deleting resources", + zap.Any("function", newFn)) // IMP - pass the oldFn, as the new/modified function is not in cache return deploy.deleteFunction(oldFn) } @@ -321,17 +335,19 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function // Executor type changed to New Deployment from something else if oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy && newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy { - log.Printf("function type changed to new deployment, creating resources: %v %v", oldFn.Metadata, newFn.Metadata) + deploy.logger.Info("function type changed to new deployment, creating resources", + zap.Any("old_function", oldFn.Metadata), + zap.Any("new_function", newFn.Metadata)) _, err := deploy.createFunction(newFn, true) if err != nil { - updateStatus(oldFn, err, "error changing the function's type to newdeploy") + deploy.updateStatus(oldFn, err, "error changing the function's type to newdeploy") } return err } fsvc, err := deploy.fsCache.GetByFunctionUID(newFn.Metadata.UID) if err != nil { - err = errors.Wrap(err, fmt.Sprintf("error updating function due to unable to find function service cache: %v", oldFn)) + err = errors.Wrapf(err, "error updating function due to unable to find function service cache: %v", oldFn) return err } @@ -349,7 +365,7 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function hpa, err := deploy.getHpa(ns, fnObjName) if err != nil { - updateStatus(oldFn, err, "error getting HPA while updating function") + deploy.updateStatus(oldFn, err, "error getting HPA while updating function") return err } @@ -375,7 +391,7 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function if hpaChanged { err := deploy.updateHpa(hpa) if err != nil { - updateStatus(oldFn, err, "error updating HPA while updating function") + deploy.updateStatus(oldFn, err, "error updating HPA while updating function") return err } } @@ -413,15 +429,15 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function env, err := deploy.fissionClient.Environments(newFn.Spec.Environment.Namespace). Get(newFn.Spec.Environment.Name) if err != nil { - updateStatus(oldFn, err, "failed to get environment while updating function") + deploy.updateStatus(oldFn, err, "failed to get environment while updating function") return err } deployLabels := deploy.getDeployLabels(oldFn, env) - log.Printf("updating %v deployment due to function %v update", fnObjName, newFn.Metadata.Name) + deploy.logger.Info("updating deployment due to function update", zap.String("deployment", fnObjName), zap.String("function", newFn.Metadata.Name)) newDeployment, err := deploy.getDeploymentSpec(newFn, env, fnObjName, deployLabels) if err != nil { - updateStatus(oldFn, err, "failed to get new deployment spec while updating function") + deploy.updateStatus(oldFn, err, "failed to get new deployment spec while updating function") return err } @@ -434,7 +450,7 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function err = deploy.updateDeployment(newDeployment, ns) if err != nil { - updateStatus(oldFn, err, "failed to update deployment while updating function") + deploy.updateStatus(oldFn, err, "failed to update deployment while updating function") return err } } @@ -506,13 +522,13 @@ func (deploy *NewDeploy) updateKubeObjRefRV(fsvc *fscache.FuncSvc, objKind strin } } fsvc.KubernetesObjects = kubeObjs - return errors.New(fmt.Sprintf("error finding kubernetes object reference with kind: %v", objKind)) + return fmt.Errorf("error finding kubernetes object reference with kind: %v", objKind) } // updateStatus is a function which updates status of update. // Current implementation only logs messages, in future it will update function status -func updateStatus(fn *crd.Function, err error, message string) { - log.Println(message, fn, err) +func (deploy *NewDeploy) updateStatus(fn *crd.Function, err error, message string) { + deploy.logger.Info("function status update", zap.Error(err), zap.Any("function", fn), zap.String("message", message)) } // IsValid does a get on the service address to ensure it's a valid service, then @@ -526,20 +542,20 @@ func (deploy *NewDeploy) IsValid(fsvc *fscache.FuncSvc) bool { _, err := deploy.kubernetesClient.CoreV1().Services(service[1]).Get(service[0], metav1.GetOptions{}) if err != nil { - log.Printf("Error validating service address for function %v: %v", fsvc.Function.Name, err) + deploy.logger.Error("error validating function service address", zap.String("function", fsvc.Function.Name), zap.Error(err)) return false } deployObj := getDeploymentObj(fsvc.KubernetesObjects) if deployObj == nil { - log.Printf("Deployment obj for function %v does not exist", fsvc.Function.Name) + deploy.logger.Error("deployment obj for function does not exist", zap.String("function", fsvc.Function.Name)) return false } currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1(). Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{}) if err != nil { - log.Printf("Error validating deployment for function %v: %v", fsvc.Function.Name, err) + deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name)) return false } @@ -560,7 +576,7 @@ func (deploy *NewDeploy) idleObjectReaper() { envs, err := deploy.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { - log.Fatalf("Failed to get environment list: %v", err) + deploy.logger.Fatal("failed to get environment list", zap.Error(err)) } envList := make(map[types.UID]struct{}) @@ -570,7 +586,7 @@ func (deploy *NewDeploy) idleObjectReaper() { funcSvcs, err := deploy.fsCache.ListOld(deploy.idlePodReapTime) if err != nil { - log.Printf("Error reaping idle pods: %v", err) + deploy.logger.Error("error reaping idle pods", zap.Error(err)) continue } @@ -582,8 +598,9 @@ func (deploy *NewDeploy) idleObjectReaper() { // For function with the environment that no longer exists, executor // scales down the deployment as usual and prints log to notify user. if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok { - log.Printf("Environment %v for function %v no longer exists", - fsvc.Environment.Metadata.Name, fsvc.Name) + deploy.logger.Error("function environment no longer exists", + zap.String("environment", fsvc.Environment.Metadata.Name), + zap.String("function", fsvc.Name)) } fn, err := deploy.fissionClient.Functions(fsvc.Function.Namespace).Get(fsvc.Function.Name) @@ -593,20 +610,20 @@ func (deploy *NewDeploy) idleObjectReaper() { if k8sErrs.IsNotFound(err) && fsvc.Executor == fscache.NEWDEPLOY { continue } - log.Printf("Error getting function: %v", fsvc.Function.Name) + deploy.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name)) continue } deployObj := getDeploymentObj(fsvc.KubernetesObjects) if deployObj == nil { - log.Printf("Error finding deployment for function %v: %v", fsvc.Function.Name, err) + deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name)) continue } currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1(). Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{}) if err != nil { - log.Printf("Error validating deployment for function %v: %v", fsvc.Function.Name, err) + deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name)) continue } @@ -617,9 +634,9 @@ func (deploy *NewDeploy) idleObjectReaper() { continue } - err = scaleDeployment(deploy.kubernetesClient, deployObj.Namespace, deployObj.Name, minScale) + err = deploy.scaleDeployment(deployObj.Namespace, deployObj.Name, minScale) if err != nil { - log.Printf("Error scaling down deployment for function %v: %v", fsvc.Function.Name, err) + deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name)) } } } @@ -635,9 +652,12 @@ func getDeploymentObj(kubeobjs []apiv1.ObjectReference) *apiv1.ObjectReference { return nil } -func scaleDeployment(client *kubernetes.Clientset, deplNS string, deplName string, replicas int32) error { - log.Printf("Scale deployment %v in namespace %v to replicas %v", deplName, deplNS, replicas) - _, err := client.ExtensionsV1beta1().Deployments(deplNS).UpdateScale(deplName, &v1beta1.Scale{ +func (deploy *NewDeploy) scaleDeployment(deplNS string, deplName string, replicas int32) error { + deploy.logger.Info("scaling deployment", + zap.String("deployment", deplName), + zap.String("namespace", deplNS), + zap.Int32("replicas", replicas)) + _, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deplNS).UpdateScale(deplName, &v1beta1.Scale{ ObjectMeta: metav1.ObjectMeta{ Name: deplName, Namespace: deplNS, diff --git a/executor/poolmgr/funcwatcher.go b/executor/poolmgr/funcwatcher.go index 33ae2f71..34700cc5 100644 --- a/executor/poolmgr/funcwatcher.go +++ b/executor/poolmgr/funcwatcher.go @@ -19,7 +19,7 @@ package poolmgr import ( "time" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -72,11 +72,15 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie // setup rolebinding is tried, if it fails, we dont return. we just log an error and move on, because : // 1. not all functions have secrets and/or configmaps, so things will work without this rolebinding in that case. // 2. on the contrary, when the route is tried, the env fetcher logs will show a 403 forbidden message and same will be relayed to executor. - err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs) + err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs) if err != nil { - log.Printf("Error : %v creating %s RoleBinding", err, fission.SecretConfigMapGetterRB) + gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", fission.SecretConfigMapGetterRB)) } else { - log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in func's ns for func : %s/%s", fission.FissionFetcherSA, envNs, fn.Metadata.Namespace, fn.Metadata.Name) + gpm.logger.Info("successfully set up rolebinding for fetcher service account for function", + zap.String("service_account", fission.FissionFetcherSA), + zap.String("service_account_namepsace", envNs), + zap.String("function_name", fn.Metadata.Name), + zap.String("function_namespace", fn.Metadata.Namespace)) } if istioEnabled { @@ -126,7 +130,11 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie // create function istio service if it does not exist _, err = kubernetesClient.CoreV1().Services(envNs).Create(&svc) if err != nil && !kerrors.IsAlreadyExists(err) { - log.Printf("Error creating function istio service: %v", err) + gpm.logger.Error("error creating istio service for function", + zap.Error(err), + zap.String("service_name", svcName), + zap.String("function_name", fn.Metadata.Name), + zap.Any("selectors", sel)) } } }, @@ -149,7 +157,11 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie // delete function istio service err := kubernetesClient.CoreV1().Services(envNs).Delete(svcName, nil) if err != nil && !kerrors.IsNotFound(err) { - log.Printf("Error deleting function istio service: %v", err) + gpm.logger.Error("error deleting istio service for function", + zap.Error(err), + zap.String("service_name", svcName), + zap.String("function_name", fn.Metadata.Name)) + } } }, @@ -178,13 +190,18 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie envNs = newFunc.Spec.Environment.Namespace } - err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB, + err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.SecretConfigMapGetterRB, newFunc.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs) + if err != nil { - log.Printf("Error : %v creating GetSecretConfigMapRoleBinding", err) + gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", fission.SecretConfigMapGetterRB)) } else { - log.Printf("Set up rolebinding for fetcher SA in func's env ns : %s, in func's ns : %s, for func : %s", newFunc.Spec.Environment.Namespace, newFunc.Metadata.Namespace, newFunc.Metadata.Name) + gpm.logger.Info("successfully set up rolebinding for fetcher service account for function", + zap.String("service_account", fission.FissionFetcherSA), + zap.String("service_account_namepsace", envNs), + zap.String("function_name", newFunc.Metadata.Name), + zap.String("function_namespace", newFunc.Metadata.Namespace)) } } }, diff --git a/executor/poolmgr/gp.go b/executor/poolmgr/gp.go index f3965d39..128478b4 100644 --- a/executor/poolmgr/gp.go +++ b/executor/poolmgr/gp.go @@ -19,7 +19,6 @@ package poolmgr import ( "context" "fmt" - "log" "math/rand" "net" "os" @@ -30,6 +29,7 @@ import ( "github.com/dchest/uniuri" multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,6 +46,7 @@ import ( type ( GenericPool struct { + logger *zap.Logger env *crd.Environment replicas int32 // num idle pods deployment *v1beta1.Deployment // kubernetes deployment @@ -83,6 +84,7 @@ type ( ) func MakeGenericPool( + logger *zap.Logger, fissionClient *crd.FissionClient, kubernetesClient *kubernetes.Clientset, env *crd.Environment, @@ -94,7 +96,9 @@ func MakeGenericPool( enableIstio bool, collectorEndpoint string) (*GenericPool, error) { - log.Printf("Creating pool for environment %v", env.Metadata) + gpLogger := logger.Named("generic_pool") + + gpLogger.Info("creating pool", zap.Any("environment", env.Metadata)) fetcherImage := os.Getenv("FETCHER_IMAGE") if len(fetcherImage) == 0 { @@ -104,6 +108,7 @@ func MakeGenericPool( // TODO: in general we need to provide the user a way to configure pools. Initial // replicas, autoscaling params, various timeouts, etc. gp := &GenericPool{ + logger: gpLogger, env: env, replicas: initialReplicas, // TODO make this an env param instead? requestChannel: make(chan *choosePodRequest), @@ -128,13 +133,12 @@ func MakeGenericPool( gp.runtimeImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY")) gp.fetcherImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("FETCHER_IMAGE_PULL_POLICY")) - log.Printf("fetcher image: %v, pull policy: %v", gp.fetcherImage, gp.fetcherImagePullPolicy) + gpLogger.Info("fetcher", zap.String("image", gp.fetcherImage), zap.Any("pull_policy", gp.fetcherImagePullPolicy)) // create fetcher SA in this ns, if not already created _, err := fission.SetupSA(gp.kubernetesClient, fission.FissionFetcherSA, gp.namespace) if err != nil { - log.Printf("Error : %v creating fetcher SA in ns : %s", err, gp.namespace) - return nil, err + return nil, errors.Wrapf(err, "error creating fetcher service account in namespace %q", gp.namespace) } // Labels for generic deployment/RS/pods. @@ -145,7 +149,7 @@ func MakeGenericPool( if err != nil { return nil, err } - log.Printf("[%v] Deployment created", env.Metadata) + gpLogger.Info("deployment created", zap.Any("environment", env.Metadata)) go gp.choosePodService() @@ -195,7 +199,7 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro for { // Retries took too long, error out. if time.Since(startTime) > gp.podReadyTimeout { - log.Printf("[%v] Erroring out, timed out", newLabels) + gp.logger.Error("timed out waiting for pod", zap.Any("labels", newLabels), zap.Duration("timeout", gp.podReadyTimeout)) return nil, errors.New("timeout: waited too long to get a ready pod") } @@ -220,7 +224,10 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro // add it to the list of ready pods readyPods = append(readyPods, &pod) } - log.Printf("[%v] found %v ready pods of %v total", newLabels, len(readyPods), len(podList.Items)) + gp.logger.Info("found ready pods", + zap.Any("labels", newLabels), + zap.Int("ready_count", len(readyPods)), + zap.Int("total", len(podList.Items))) // If there are no ready pods, wait and retry. if len(readyPods) == 0 { @@ -241,14 +248,14 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro // modified, this should fail; in that case just // retry. chosenPod.ObjectMeta.Labels = newLabels - log.Printf("relabeling pod: [%v]", chosenPod.ObjectMeta.Name) + gp.logger.Info("relabeling pod", zap.String("pod", chosenPod.ObjectMeta.Name)) _, err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Update(chosenPod) if err != nil { - log.Printf("failed to relabel pod [%v]: %v", chosenPod.ObjectMeta.Name, err) + gp.logger.Error("failed to relabel pod", zap.Error(err), zap.String("pod", chosenPod.ObjectMeta.Name)) continue } } - log.Printf("Chosen pod: %v (in %v)", chosenPod.ObjectMeta.Name, time.Since(startTime)) + gp.logger.Info("chose pod", zap.String("pod", chosenPod.ObjectMeta.Name), zap.Duration("elapsed_time", time.Since(startTime))) return chosenPod, nil } } @@ -267,7 +274,7 @@ func (gp *GenericPool) scheduleDeletePod(name string) { // The sleep allows debugging or collecting logs from the pod before it's // cleaned up. (We need a better solutions for both those things; log // aggregation and storage will help.) - log.Printf("Error in pod '%v', scheduling cleanup", name) + gp.logger.Error("error in pod - scheduling cleanup", zap.String("pod", name)) // Ignore sleep here if istio feature is enabled, function pod // will be deleted after 6 mins (terminationGracePeriodSeconds). if !gp.useIstio { @@ -319,7 +326,7 @@ func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metada // tell fetcher to get the function. fetcherUrl := gp.getSpecializeUrl(podIP) - log.Printf("[%v] calling fetcher to copy function with fetcher url: %v", metadata.Name, fetcherUrl) + gp.logger.Info("calling fetcher to copy function", zap.String("function", metadata.Name), zap.String("url", fetcherUrl)) fn, err := gp.fissionClient. Functions(metadata.Namespace). @@ -356,9 +363,9 @@ func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metada }, } - log.Printf("[%v] specializing pod", metadata.Name) + gp.logger.Info("specializing pod", zap.String("function", metadata.Name)) - err = fetcherClient.MakeClient(fetcherUrl).Specialize(ctx, &specializeReq) + err = fetcherClient.MakeClient(gp.logger, fetcherUrl).Specialize(ctx, &specializeReq) if err != nil { return err } @@ -550,7 +557,7 @@ func (gp *GenericPool) createPool() error { depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment) if err != nil { - log.Printf("Error creating deployment for %s in kubernetes, err: %v", deployment.Name, err) + gp.logger.Error("error creating deployment in kubernetes", zap.Error(err), zap.String("deployment", deployment.Name)) return err } gp.deployment = depl @@ -564,11 +571,9 @@ func (gp *GenericPool) waitForReadyPod() error { depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get( gp.deployment.ObjectMeta.Name, metav1.GetOptions{}) if err != nil { - err = errors.Wrap(err, fmt.Sprintf( - "Error waiting for ready pod of deployment %v in namespace %v", - gp.deployment.ObjectMeta.Name, gp.namespace)) - log.Print(err) - return err + e := "error waiting for ready pod for deployment" + gp.logger.Error(e, zap.String("deployment", gp.deployment.ObjectMeta.Name), zap.String("namespace", gp.namespace)) + return fmt.Errorf("%s %q in namespace %q", e, gp.deployment.ObjectMeta.Name, gp.namespace) } gp.deployment = depl @@ -582,7 +587,7 @@ func (gp *GenericPool) waitForReadyPod() error { gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(), }) if err != nil { - log.Printf("Error getting pod list after timeout waiting for ready pod: %v", err) + gp.logger.Error("error getting pod list after timeout waiting for ready pod", zap.Error(err)) } // Since even single pod is not ready, choosing the first pod to inspect is a good approximation. In future this can be done better @@ -593,9 +598,8 @@ func (gp *GenericPool) waitForReadyPod() error { multierr = multierror.Append(multierr, errors.New(fmt.Sprintf("%v: %v", cStatus.State.Waiting.Reason, cStatus.State.Waiting.Message))) } } - return errors.Errorf( - "Timeout: waited too long for pod of deployment %v in namespace %v to be ready, error: %v", - gp.deployment.ObjectMeta.Name, gp.namespace, multierr) + return errors.Wrapf(multierr, "Timeout: waited too long for pod of deployment %v in namespace %v to be ready", + gp.deployment.ObjectMeta.Name, gp.namespace) } time.Sleep(1000 * time.Millisecond) } @@ -623,7 +627,7 @@ func (gp *GenericPool) createSvc(name string, labels map[string]string) (*apiv1. } func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*fscache.FuncSvc, error) { - log.Printf("[%v] Choosing pod from pool", m.Name) + gp.logger.Info("choosing pod from pool", zap.String("function", m.Name)) newLabels := gp.labelsForFunction(m) if gp.useIstio { @@ -677,7 +681,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f gp.scheduleDeletePod(pod.ObjectMeta.Name) return nil, err } - log.Printf("Specialized pod: %v", pod.ObjectMeta.Name) + gp.logger.Info("specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name)) var svcHost string if gp.useSvc && !gp.useIstio { @@ -694,7 +698,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f } if svc.ObjectMeta.Name != svcName { gp.scheduleDeletePod(pod.ObjectMeta.Name) - return nil, errors.New(fmt.Sprintf("sanity check failed for svc %v", svc.ObjectMeta.Name)) + return nil, errors.Errorf("sanity check failed for svc %v", svc.ObjectMeta.Name) } // the fission router isn't in the same namespace, so return a @@ -704,7 +708,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f svc := fission.GetFunctionIstioServiceName(m.Name, m.Namespace) svcHost = fmt.Sprintf("%v.%v:8888", svc, gp.namespace) } else { - log.Printf("Using pod IP for specialized pod") + gp.logger.Info("using pod IP for specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name)) svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP) } @@ -746,7 +750,10 @@ func (gp *GenericPool) destroy() error { err := gp.kubernetesClient.ExtensionsV1beta1(). Deployments(gp.namespace).Delete(gp.deployment.ObjectMeta.Name, &delOpt) if err != nil { - log.Printf("Error destroying deployment: %v", err) + gp.logger.Error("error destroying deployment", + zap.Error(err), + zap.String("deployment_name", gp.deployment.ObjectMeta.Name), + zap.String("deployment_namespace", gp.namespace)) return err } return nil diff --git a/executor/poolmgr/gpm.go b/executor/poolmgr/gpm.go index a38bb8fd..9a1279f7 100644 --- a/executor/poolmgr/gpm.go +++ b/executor/poolmgr/gpm.go @@ -18,12 +18,12 @@ package poolmgr import ( "context" - "log" "os" "strconv" "strings" "time" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -45,6 +45,8 @@ const ( type ( GenericPoolManager struct { + logger *zap.Logger + pools map[string]*GenericPool kubernetesClient *kubernetes.Clientset namespace string @@ -78,18 +80,22 @@ type ( ) func MakeGenericPoolManager( + logger *zap.Logger, fissionClient *crd.FissionClient, kubernetesClient *kubernetes.Clientset, functionNamespace string, instanceId string) *GenericPoolManager { + gpmLogger := logger.Named("generic_pool_manager") + gpm := &GenericPoolManager{ + logger: gpmLogger, pools: make(map[string]*GenericPool), kubernetesClient: kubernetesClient, namespace: functionNamespace, fissionClient: fissionClient, functionEnv: cache.MakeCache(10*time.Second, 0), - fsCache: fscache.MakeFunctionServiceCache(), + fsCache: fscache.MakeFunctionServiceCache(gpmLogger), instanceId: instanceId, requestChannel: make(chan *request), idlePodReapTime: 2 * time.Minute, @@ -101,7 +107,7 @@ func MakeGenericPoolManager( if len(os.Getenv("ENABLE_ISTIO")) > 0 { istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO")) if err != nil { - log.Println("Failed to parse ENABLE_ISTIO") + gpmLogger.Info("failed to parse ENABLE_ISTIO") } gpm.enableIstio = istio } @@ -142,7 +148,7 @@ func (gpm *GenericPoolManager) service() { ns = req.env.Metadata.Namespace } - pool, err = MakeGenericPool( + pool, err = MakeGenericPool(gpm.logger, gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize, ns, gpm.namespace, gpm.fsCache, gpm.instanceId, gpm.enableIstio, gpm.collectorEndpoint) @@ -163,7 +169,7 @@ func (gpm *GenericPoolManager) service() { if !ok || poolsize == 0 { // Env no longer exists or pool size changed to zero - log.Printf("Destroying generic pool for environment %v", pool.env.Metadata) + gpm.logger.Info("eestroying generic pool", zap.Any("environment", pool.env.Metadata)) delete(gpm.pools, key) // and delete the pool asynchronously. @@ -195,7 +201,7 @@ func (gpm *GenericPoolManager) CleanupPools(envs []crd.Environment) { func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) { // from Func -> get Env - log.Printf("[%v] getting environment for function", metadata.Name) + gpm.logger.Info("getting environment for function", zap.String("function", metadata.Name)) env, err := gpm.getFunctionEnv(metadata) if err != nil { return nil, err @@ -207,7 +213,7 @@ func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1. } // from GenericPool -> get one function container // (this also adds to the cache) - log.Printf("[%v] getting function service from pool", metadata.Name) + gpm.logger.Info("getting function service from pool", zap.String("function", metadata.Name)) return pool.GetFuncSvc(ctx, metadata) } @@ -228,7 +234,7 @@ func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Enviro } // Get env from metadata - log.Printf("[%v] getting env", m) + gpm.logger.Info("getting env", zap.Any("function", m)) env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name) if err != nil { return nil, err @@ -247,11 +253,11 @@ func (gpm *GenericPoolManager) eagerPoolCreator() { envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { if fission.IsNetworkError(err) { - log.Printf("Encountered network error, retrying: %v", err) + gpm.logger.Error("encountered network error, retrying", zap.Error(err)) time.Sleep(5 * time.Second) continue } - log.Fatalf("Failed to get environment list: %v", err) + gpm.logger.Fatal("failed to get environment list", zap.Error(err)) } // Create pools for all envs. TODO: we should make this a bit less eager, only @@ -264,7 +270,7 @@ func (gpm *GenericPoolManager) eagerPoolCreator() { if gpm.getEnvPoolsize(&env) > 0 { _, err := gpm.GetPool(&envs.Items[i]) if err != nil { - log.Printf("eager-create pool failed: %v", err) + gpm.logger.Error("eager-create pool failed", zap.Error(err)) } } } @@ -292,7 +298,7 @@ func (gpm *GenericPoolManager) IsValid(fsvc *fscache.FuncSvc) bool { if obj.Kind == "pod" { pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{}) if err == nil && strings.Contains(fsvc.Address, pod.Status.PodIP) && fission.IsReadyPod(pod) { - log.Printf("Valid pod address : %s", fsvc.Address) + gpm.logger.Info("valid pod address", zap.String("address", fsvc.Address)) return true } } @@ -309,7 +315,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() { envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { - log.Fatalf("Failed to get environment list: %v", err) + gpm.logger.Fatal("failed to get environment list", zap.Error(err)) } envList := make(map[types.UID]struct{}) @@ -319,7 +325,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() { funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime) if err != nil { - log.Printf("Error reaping idle pods: %v", err) + gpm.logger.Error("error reaping idle pods", zap.Error(err)) continue } @@ -331,8 +337,9 @@ func (gpm *GenericPoolManager) idleObjectReaper() { // For function with the environment that no longer exists, executor // cleanups the idle pod as usual and prints log to notify user. if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok { - log.Printf("Environment %v for function %v no longer exists", - fsvc.Environment.Metadata.Name, fsvc.Name) + gpm.logger.Info("function environment no longer exists", + zap.String("environment", fsvc.Environment.Metadata.Name), + zap.String("function", fsvc.Name)) } if fsvc.Environment.Spec.AllowedFunctionsPerContainer == fission.AllowedFunctionsPerContainerInfinite { @@ -341,7 +348,9 @@ func (gpm *GenericPoolManager) idleObjectReaper() { deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime) if err != nil { - log.Printf("Error deleting Kubernetes objects for fsvc '%v': %v", fsvc, err) + gpm.logger.Error("error deleting Kubernetes objects for function service", + zap.Error(err), + zap.Any("service", fsvc)) } if !deleted { @@ -349,7 +358,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() { } for _, kubeobj := range fsvc.KubernetesObjects { - reaper.CleanupKubeObject(gpm.kubernetesClient, &kubeobj) + reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &kubeobj) } } } diff --git a/executor/poolmgr/packagewatcher.go b/executor/poolmgr/packagewatcher.go index 9992299f..38d62cd8 100644 --- a/executor/poolmgr/packagewatcher.go +++ b/executor/poolmgr/packagewatcher.go @@ -19,7 +19,7 @@ package poolmgr import ( "time" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" @@ -39,7 +39,9 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien k8sCache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { pkg := obj.(*crd.Package) - log.Printf("List watch for package reported a new package addition: %s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace) + gpm.logger.Info("list watch for package reported a new package addition", + zap.String("package_name", pkg.Metadata.Name), + zap.String("package_namepsace", pkg.Metadata.Namespace)) // create or update role-binding for fetcher sa in env ns to be able to get the pkg contents from pkg namespace envNs := fissionfnNamespace @@ -49,13 +51,21 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien // here, we return if we hit an error during rolebinding setup. this is because this rolebinding is mandatory for // every function's package to be loaded into its env. without that, there's no point to move forward. - err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs) + err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs) if err != nil { - log.Printf("Error creating %s for package: %s.%s, err: %v", fission.PackageGetterRB, pkg.Metadata.Name, pkg.Metadata.Namespace, err) + gpm.logger.Error("error creating rolebinding for package", + zap.Error(err), + zap.String("role_binding", fission.PackageGetterRB), + zap.String("package_name", pkg.Metadata.Name), + zap.String("package_namespace", pkg.Metadata.Namespace)) return } - log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, pkg.Metadata.Namespace, pkg.Metadata.Name) + gpm.logger.Info("successfully set up rolebinding for fetcher service account", + zap.String("service_account", fission.FissionFetcherSA), + zap.String("service_account_namespace", envNs), + zap.String("package_name", pkg.Metadata.Name), + zap.String("package_namespace", pkg.Metadata.Namespace)) }, UpdateFunc: func(oldObj, newObj interface{}) { @@ -75,14 +85,23 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien envNs = newPkg.Spec.Environment.Namespace } - err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB, + err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.PackageGetterRB, newPkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs) if err != nil { - log.Printf("Error : %v updating %s for package: %s.%s", err, fission.PackageGetterRB, newPkg.Metadata.Name, newPkg.Metadata.Namespace) + gpm.logger.Error("error updating rolebinding for package", + zap.Error(err), + zap.String("role_binding", fission.PackageGetterRB), + zap.String("package_name", newPkg.Metadata.Name), + zap.String("package_namespace", newPkg.Metadata.Namespace)) return } - log.Printf("Updated rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, newPkg.Metadata.Namespace, newPkg.Metadata.Name) + + gpm.logger.Info("successfully updated rolebinding for fetcher service account", + zap.String("service_account", fission.FissionFetcherSA), + zap.String("service_account_namespace", envNs), + zap.String("package_name", newPkg.Metadata.Name), + zap.String("package_namespace", newPkg.Metadata.Namespace)) } }, }) diff --git a/executor/reaper/reaper.go b/executor/reaper/reaper.go index e69174c0..baf6ea3c 100644 --- a/executor/reaper/reaper.go +++ b/executor/reaper/reaper.go @@ -17,11 +17,10 @@ limitations under the License. package reaper import ( - "fmt" - "log" "strings" "time" + "go.uber.org/zap" apiv1 "k8s.io/api/core/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -36,24 +35,24 @@ var ( ) // CleanupOldExecutorObjects cleans up resources created by old executor instances -func CleanupOldExecutorObjects(kubernetesClient *kubernetes.Clientset, instanceId string) { +func CleanupOldExecutorObjects(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, instanceId string) { go func() { - err := cleanup(kubernetesClient, instanceId) + err := cleanup(logger, kubernetesClient, instanceId) if err != nil { // TODO retry reaper; logged and ignored for now - log.Printf("Failed to reaper: %v", err) + logger.Error("Failed to cleanup old executor objects", zap.Error(err)) } }() } -func cleanup(client *kubernetes.Clientset, instanceId string) error { +func cleanup(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error { - err := cleanupServices(client, instanceId) + err := cleanupServices(logger, client, instanceId) if err != nil { return err } - err = cleanupHpa(client, instanceId) + err = cleanupHpa(logger, client, instanceId) if err != nil { return err } @@ -61,7 +60,7 @@ func cleanup(client *kubernetes.Clientset, instanceId string) error { // Deployments are used for idle pools and can be cleaned up // immediately. (We should "adopt" these instead of creating // a new pool.) - err = cleanupDeployments(client, instanceId) + err = cleanupDeployments(logger, client, instanceId) if err != nil { return err } @@ -73,7 +72,7 @@ func cleanup(client *kubernetes.Clientset, instanceId string) error { // time. time.Sleep(6 * time.Minute) - err = cleanupPods(client, instanceId) + err = cleanupPods(logger, client, instanceId) if err != nil { return err } @@ -82,31 +81,39 @@ func cleanup(client *kubernetes.Clientset, instanceId string) error { } // CleanupKubeObject deletes given kubernetes object -func CleanupKubeObject(kubeClient *kubernetes.Clientset, kubeobj *apiv1.ObjectReference) { +func CleanupKubeObject(logger *zap.Logger, kubeClient *kubernetes.Clientset, kubeobj *apiv1.ObjectReference) { switch strings.ToLower(kubeobj.Kind) { case "pod": err := kubeClient.CoreV1().Pods(kubeobj.Namespace).Delete(kubeobj.Name, nil) - logErr(fmt.Sprintf("cleaning up pod %v ", kubeobj.Name), err) + if err != nil { + logger.Error("error cleaning up pod", zap.Error(err), zap.String("pod", kubeobj.Name)) + } case "service": err := kubeClient.CoreV1().Services(kubeobj.Namespace).Delete(kubeobj.Name, nil) - logErr(fmt.Sprintf("cleaning up service %v ", kubeobj.Name), err) + if err != nil { + logger.Error("error cleaning up service", zap.Error(err), zap.String("service", kubeobj.Name)) + } case "deployment": err := kubeClient.ExtensionsV1beta1().Deployments(kubeobj.Namespace).Delete(kubeobj.Name, &delOpt) - logErr(fmt.Sprintf("cleaning up deployment %v ", kubeobj.Name), err) + if err != nil { + logger.Error("error cleaning up deployment", zap.Error(err), zap.String("deployment", kubeobj.Name)) + } case "horizontalpodautoscaler": err := kubeClient.AutoscalingV1().HorizontalPodAutoscalers(kubeobj.Namespace).Delete(kubeobj.Name, nil) - logErr(fmt.Sprintf("cleaning up horizontalpodautoscaler %v ", kubeobj.Name), err) + if err != nil { + logger.Error("error cleaning up horizontalpodautoscaler", zap.Error(err), zap.String("horizontalpodautoscaler", kubeobj.Name)) + } default: - log.Printf("There was an error identifying the object type: %v for obj: %v", kubeobj.Kind, kubeobj) + logger.Error("Could not identifying the object type to clean up", zap.String("type", kubeobj.Kind), zap.Any("object", kubeobj)) } } -func cleanupDeployments(client *kubernetes.Clientset, instanceId string) error { +func cleanupDeployments(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error { deploymentList, err := client.ExtensionsV1beta1().Deployments(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) if err != nil { return err @@ -114,24 +121,34 @@ func cleanupDeployments(client *kubernetes.Clientset, instanceId string) error { for _, dep := range deploymentList.Items { id, ok := dep.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL] if ok && id != instanceId { - log.Printf("Cleaning up deployment %v", dep.ObjectMeta.Name) + logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name)) err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt) - logErr("cleaning up deployment", err) + if err != nil { + logger.Error("error cleaning up deployment", + zap.Error(err), + zap.String("deployment_name", dep.ObjectMeta.Name), + zap.String("deployment_namespace", dep.ObjectMeta.Namespace)) + } // ignore err } // Backward compatibility with older label name pid, pok := dep.ObjectMeta.Labels[fission.POOLMGR_INSTANCEID_LABEL] if pok && pid != instanceId { - log.Printf("Cleaning up deployment %v", dep.ObjectMeta.Name) + logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name)) err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt) - logErr("cleaning up deployment", err) + if err != nil { + logger.Error("error cleaning up deployment", + zap.Error(err), + zap.String("deployment_name", dep.ObjectMeta.Name), + zap.String("deployment_namespace", dep.ObjectMeta.Namespace)) + } // ignore err } } return nil } -func cleanupPods(client *kubernetes.Clientset, instanceId string) error { +func cleanupPods(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error { podList, err := client.CoreV1().Pods(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) if err != nil { return err @@ -139,17 +156,27 @@ func cleanupPods(client *kubernetes.Clientset, instanceId string) error { for _, pod := range podList.Items { id, ok := pod.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL] if ok && id != instanceId { - log.Printf("Cleaning up pod %v", pod.ObjectMeta.Name) + logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name)) err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil) - logErr("cleaning up pod", err) + if err != nil { + logger.Error("error cleaning up pod", + zap.Error(err), + zap.String("pod_name", pod.ObjectMeta.Name), + zap.String("pod_namespace", pod.ObjectMeta.Namespace)) + } // ignore err } // Backward compatibility with older label name pid, pok := pod.ObjectMeta.Labels[fission.POOLMGR_INSTANCEID_LABEL] if pok && pid != instanceId { - log.Printf("Cleaning up pod %v", pod.ObjectMeta.Name) + logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name)) err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil) - logErr("cleaning up pod", err) + if err != nil { + logger.Error("error cleaning up pod", + zap.Error(err), + zap.String("pod_name", pod.ObjectMeta.Name), + zap.String("pod_namespace", pod.ObjectMeta.Namespace)) + } // ignore err } @@ -157,7 +184,7 @@ func cleanupPods(client *kubernetes.Clientset, instanceId string) error { return nil } -func cleanupServices(client *kubernetes.Clientset, instanceId string) error { +func cleanupServices(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error { svcList, err := client.CoreV1().Services(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) if err != nil { return err @@ -165,16 +192,21 @@ func cleanupServices(client *kubernetes.Clientset, instanceId string) error { for _, svc := range svcList.Items { id, ok := svc.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL] if ok && id != instanceId { - log.Printf("Cleaning up svc %v", svc.ObjectMeta.Name) + logger.Info("cleaning up service", zap.String("service", svc.ObjectMeta.Name)) err := client.CoreV1().Services(svc.ObjectMeta.Namespace).Delete(svc.ObjectMeta.Name, nil) - logErr("cleaning up service", err) + if err != nil { + logger.Error("error cleaning up service", + zap.Error(err), + zap.String("service_name", svc.ObjectMeta.Name), + zap.String("service_namespace", svc.ObjectMeta.Namespace)) + } // ignore err } } return nil } -func cleanupHpa(client *kubernetes.Clientset, instanceId string) error { +func cleanupHpa(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error { hpaList, err := client.AutoscalingV1().HorizontalPodAutoscalers(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) if err != nil { return err @@ -183,9 +215,15 @@ func cleanupHpa(client *kubernetes.Clientset, instanceId string) error { for _, hpa := range hpaList.Items { id, ok := hpa.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL] if ok && id != instanceId { - log.Printf("Cleaning up HPA %v", hpa.ObjectMeta.Name) + logger.Info("cleaning up HPA", zap.String("hpa", hpa.ObjectMeta.Name)) err := client.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Delete(hpa.ObjectMeta.Name, nil) - logErr("cleaning up HPA", err) + if err != nil { + logger.Error("error cleaning up HPA", + zap.Error(err), + zap.String("hpa_name", hpa.ObjectMeta.Name), + zap.String("hpa_namespace", hpa.ObjectMeta.Namespace)) + } + // ignore err } } @@ -193,22 +231,16 @@ func cleanupHpa(client *kubernetes.Clientset, instanceId string) error { } -func logErr(msg string, err error) { - if err != nil { - log.Printf("Error %v: %v", msg, err) - } -} - // CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or // deletes the rolebindings completely if there are no Service Accounts in a rolebinding object. -func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) { +func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) { for { - log.Println("Starting cleanupRoleBindings cycle") + logger.Info("starting cleanupRoleBindings cycle") // get all rolebindings ( just to be efficient, one call to kubernetes ) rbList, err := client.RbacV1beta1().RoleBindings(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) if err != nil { // something wrong, but next iteration hopefully succeeds - log.Printf("Error listing rolebindings in all ns, err: %v", err) + logger.Error("error listing role bindings in all namespaces", zap.Error(err)) continue } @@ -228,7 +260,7 @@ func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.Fissio // we can list the functions once per role-binding. funcList, err := fissionClient.Functions(roleBinding.Namespace).List(meta_v1.ListOptions{}) if err != nil { - log.Printf("Error fetching environment list in : %s", roleBinding.Namespace) + logger.Error("error fetching environment list in namespace", zap.Error(err), zap.String("namespace", roleBinding.Namespace)) continue } @@ -275,7 +307,7 @@ func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.Fissio // check if there is an env obj in saNs envList, err := fissionClient.Environments(saNs).List(meta_v1.ListOptions{}) if err != nil { - log.Printf("Error fetching environment list in : %s", saNs) + logger.Error("error fetching environment list in service account namespace", zap.Error(err), zap.String("namespace", saNs)) continue } @@ -313,15 +345,21 @@ func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.Fissio // finally, make a call to RemoveSAFromRoleBindingWithRetries for all the service accounts that need to be removed // for the role-binding in this iteration if len(saToRemove) != 0 { - log.Printf("saToRemove : %v for rolebinding : %s.%s", saToRemove, roleBinding.Name, roleBinding.Namespace) + logger.Info("removing service accounts from role binding", + zap.Any("service_accounts", saToRemove), + zap.String("role_binding_name", roleBinding.Name), + zap.String("role_binding_namespace", roleBinding.Namespace)) // call this once in the end for each role-binding - err = fission.RemoveSAFromRoleBindingWithRetries(client, roleBinding.Name, roleBinding.Namespace, saToRemove) + err = fission.RemoveSAFromRoleBindingWithRetries(logger, client, roleBinding.Name, roleBinding.Namespace, saToRemove) if err != nil { // if there's an error, we just log it and proceed with the next role-binding, hoping that this role-binding // will be processed in next iteration. - log.Printf("Error removing SA : %v from rolebinding : %s.%s, err: %v", saToRemove, roleBinding.Name, - roleBinding.Namespace, err) + logger.Info("error removing service account from role binding", + zap.Error(err), + zap.Any("service_accounts", saToRemove), + zap.String("role_binding_name", roleBinding.Name), + zap.String("role_binding_namespace", roleBinding.Namespace)) } } } diff --git a/fission-bundle/main.go b/fission-bundle/main.go index 2d27d938..7ea36de4 100644 --- a/fission-bundle/main.go +++ b/fission-bundle/main.go @@ -6,10 +6,10 @@ import ( "os" "strconv" + docopt "github.com/docopt/docopt-go" "go.opencensus.io/exporter/jaeger" "go.opencensus.io/trace" - - docopt "github.com/docopt/docopt-go" + "go.uber.org/zap" "github.com/fission/fission" "github.com/fission/fission/buildermgr" @@ -22,66 +22,66 @@ import ( "github.com/fission/fission/timer" ) -func runController(port int) { - controller.Start(port, false) - log.Fatalf("Error: Controller exited.") +func runController(logger *zap.Logger, port int) { + controller.Start(logger, port, false) + logger.Fatal("controller exited") } -func runRouter(port int, executorUrl string) { - router.Start(port, executorUrl) - log.Fatalf("Error: Router exited.") +func runRouter(logger *zap.Logger, port int, executorUrl string) { + router.Start(logger, port, executorUrl) + logger.Fatal("router exited") } -func runExecutor(port int, fissionNamespace, functionNamespace, envBuilderNamespace string) { - err := executor.StartExecutor(fissionNamespace, functionNamespace, envBuilderNamespace, port) +func runExecutor(logger *zap.Logger, port int, fissionNamespace, functionNamespace, envBuilderNamespace string) { + err := executor.StartExecutor(logger, fissionNamespace, functionNamespace, envBuilderNamespace, port) if err != nil { - log.Fatalf("Error starting executor: %v", err) + logger.Fatal("error starting executor", zap.Error(err)) } } -func runKubeWatcher(routerUrl string) { - err := kubewatcher.Start(routerUrl) +func runKubeWatcher(logger *zap.Logger, routerUrl string) { + err := kubewatcher.Start(logger, routerUrl) if err != nil { - log.Fatalf("Error starting kubewatcher: %v", err) + logger.Fatal("error starting kubewatcher", zap.Error(err)) } } -func runTimer(routerUrl string) { - err := timer.Start(routerUrl) +func runTimer(logger *zap.Logger, routerUrl string) { + err := timer.Start(logger, routerUrl) if err != nil { - log.Fatalf("Error starting timer: %v", err) + logger.Fatal("error starting timer", zap.Error(err)) } } -func runMessageQueueMgr(routerUrl string) { - err := messagequeue.Start(routerUrl) +func runMessageQueueMgr(logger *zap.Logger, routerUrl string) { + err := messagequeue.Start(logger, routerUrl) if err != nil { - log.Fatalf("Error starting timer: %v", err) + logger.Fatal("error starting message queue manager", zap.Error(err)) } } -func runStorageSvc(port int, filePath string) { +func runStorageSvc(logger *zap.Logger, port int, filePath string) { subdir := os.Getenv("SUBDIR") if len(subdir) == 0 { subdir = "fission-functions" } enableArchivePruner := true - storagesvc.RunStorageService(storagesvc.StorageTypeLocal, + storagesvc.RunStorageService(logger, storagesvc.StorageTypeLocal, filePath, subdir, port, enableArchivePruner) } -func runBuilderMgr(storageSvcUrl string, envBuilderNamespace string) { - err := buildermgr.Start(storageSvcUrl, envBuilderNamespace) +func runBuilderMgr(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) { + err := buildermgr.Start(logger, storageSvcUrl, envBuilderNamespace) if err != nil { - log.Fatalf("Error starting buildermgr: %v", err) + logger.Fatal("error starting builder manager", zap.Error(err)) } } -func getPort(portArg interface{}) int { +func getPort(logger *zap.Logger, portArg interface{}) int { portArgStr := portArg.(string) port, err := strconv.Atoi(portArgStr) if err != nil { - log.Fatalf("Error: invalid port number '%v'", portArgStr) + logger.Fatal("invalid port number", zap.Error(err), zap.String("port", portArgStr)) } return port } @@ -94,10 +94,10 @@ func getStringArgWithDefault(arg interface{}, defaultValue string) string { } } -func registerTraceExporter(arguments map[string]interface{}) error { +func registerTraceExporter(logger *zap.Logger, arguments map[string]interface{}) error { collectorEndpoint := getStringArgWithDefault(arguments["--collectorEndpoint"], "") if collectorEndpoint == "" { - log.Print("Skipping trace exporter registration") + logger.Info("skipping trace exporter registration") return nil } @@ -193,15 +193,21 @@ Options: --builderMgr Start builder manager. --version Print version information ` + logger, err := zap.NewProduction() + if err != nil { + log.Fatalf("can't initialize zap logger: %v", err) + } + defer logger.Sync() + version := fmt.Sprintf("Fission Bundle Version: %v", fission.BuildInfo().String()) arguments, err := docopt.Parse(usage, nil, true, version, false) if err != nil { - log.Fatalf("Error: %v", err) + logger.Fatal("Could not parse command line arguments", zap.Error(err)) } - err = registerTraceExporter(arguments) + err = registerTraceExporter(logger, arguments) if err != nil { - log.Fatalf("Error: %v", err) + logger.Fatal("Could not register trace exporter", zap.Error(err), zap.Any("argument", arguments)) } functionNs := getStringArgWithDefault(arguments["--namespace"], "fission-function") @@ -213,40 +219,40 @@ Options: storageSvcUrl := getStringArgWithDefault(arguments["--storageSvcUrl"], "http://storagesvc.fission") if arguments["--controllerPort"] != nil { - port := getPort(arguments["--controllerPort"]) - runController(port) + port := getPort(logger, arguments["--controllerPort"]) + runController(logger, port) } if arguments["--routerPort"] != nil { - port := getPort(arguments["--routerPort"]) - runRouter(port, executorUrl) + port := getPort(logger, arguments["--routerPort"]) + runRouter(logger, port, executorUrl) } if arguments["--executorPort"] != nil { - port := getPort(arguments["--executorPort"]) - runExecutor(port, fissionNs, functionNs, envBuilderNs) + port := getPort(logger, arguments["--executorPort"]) + runExecutor(logger, port, fissionNs, functionNs, envBuilderNs) } if arguments["--kubewatcher"] == true { - runKubeWatcher(routerUrl) + runKubeWatcher(logger, routerUrl) } if arguments["--timer"] == true { - runTimer(routerUrl) + runTimer(logger, routerUrl) } if arguments["--mqt"] == true { - runMessageQueueMgr(routerUrl) + runMessageQueueMgr(logger, routerUrl) } if arguments["--builderMgr"] == true { - runBuilderMgr(storageSvcUrl, envBuilderNs) + runBuilderMgr(logger, storageSvcUrl, envBuilderNs) } if arguments["--storageServicePort"] != nil { - port := getPort(arguments["--storageServicePort"]) + port := getPort(logger, arguments["--storageServicePort"]) filePath := arguments["--filePath"].(string) - runStorageSvc(port, filePath) + runStorageSvc(logger, port, filePath) } select {} diff --git a/fission/logdb/influxdb.go b/fission/logdb/influxdb.go index 0dacf443..07cbeabf 100644 --- a/fission/logdb/influxdb.go +++ b/fission/logdb/influxdb.go @@ -19,7 +19,6 @@ package logdb import ( "encoding/json" "fmt" - "log" "net/http" "net/url" "sort" @@ -30,6 +29,7 @@ import ( influxdbClient "github.com/influxdata/influxdb/client/v2" "github.com/fission/fission" + "github.com/fission/fission/fission/log" ) const ( diff --git a/fission/logdb/logdb.go b/fission/logdb/logdb.go index 71b90eec..e5ff6ff9 100644 --- a/fission/logdb/logdb.go +++ b/fission/logdb/logdb.go @@ -17,9 +17,8 @@ limitations under the License. package logdb import ( + "fmt" "time" - - log "github.com/sirupsen/logrus" ) const ( @@ -55,6 +54,5 @@ func GetLogDB(dbType string, serverURL string) (LogDatabase, error) { case INFLUXDB: return NewInfluxDB(serverURL) } - log.Fatalf("Log database type is incorrect, now only support %s", INFLUXDB) - return nil, nil + return nil, fmt.Errorf("log database type is incorrect, now only support %s", INFLUXDB) } diff --git a/fission/recorder.go b/fission/recorder.go index f439ea2d..71034339 100644 --- a/fission/recorder.go +++ b/fission/recorder.go @@ -18,7 +18,6 @@ package main import ( "fmt" - "log" "os" "strings" "text/tabwriter" @@ -29,6 +28,7 @@ import ( "github.com/fission/fission" "github.com/fission/fission/crd" + "github.com/fission/fission/fission/log" "github.com/fission/fission/fission/util" ) diff --git a/fission/records.go b/fission/records.go index 73205596..525a4ced 100644 --- a/fission/records.go +++ b/fission/records.go @@ -18,12 +18,12 @@ package main import ( "fmt" - "log" "os" "text/tabwriter" "github.com/urfave/cli" + "github.com/fission/fission/fission/log" "github.com/fission/fission/fission/util" "github.com/fission/fission/redis/build/gen" ) @@ -31,7 +31,7 @@ import ( func recordsView(c *cli.Context) error { var verbosity int if c.Bool("v") && c.Bool("vv") { - log.Fatal("Conflicting verbosity levels, use either --v or --vv") + log.Fatal("conflicting verbosity levels, use either --v or --vv") } if c.Bool("v") { verbosity = 1 @@ -47,7 +47,7 @@ func recordsView(c *cli.Context) error { //Refuse multiple filters for now if multipleFiltersSpecified(function, trigger, from+to) { - log.Fatal("Maximum of one filter is currently supported, either --function, --trigger, or --from,--to") + log.Fatal("maximum of one filter is currently supported, either --function, --trigger, or --from,--to") } if len(function) != 0 { diff --git a/fission/replay.go b/fission/replay.go index e6868188..e51a8416 100644 --- a/fission/replay.go +++ b/fission/replay.go @@ -18,10 +18,10 @@ package main import ( "fmt" - "log" "os" "text/tabwriter" + "github.com/fission/fission/fission/log" "github.com/fission/fission/fission/util" "github.com/urfave/cli" ) diff --git a/fission/support/resources/crd.go b/fission/support/resources/crd.go index b2286a68..ccb8521d 100644 --- a/fission/support/resources/crd.go +++ b/fission/support/resources/crd.go @@ -17,13 +17,14 @@ limitations under the License. package resources import ( - "log" + "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" "github.com/fission/fission/controller/client" "github.com/fission/fission/crd" + "github.com/fission/fission/fission/log" ) const ( @@ -52,7 +53,7 @@ func (res CrdDumper) Dump(dumpDir string) { case CrdEnvironment: items, err := res.client.EnvironmentList(metav1.NamespaceAll) if err != nil { - log.Printf("Error getting %v list: %v", res.crdType, err) + log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err)) return } @@ -64,7 +65,7 @@ func (res CrdDumper) Dump(dumpDir string) { case CrdFunction: items, err := res.client.FunctionList(metav1.NamespaceAll) if err != nil { - log.Printf("Error getting %v list: %v", res.crdType, err) + log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err)) return } @@ -76,7 +77,7 @@ func (res CrdDumper) Dump(dumpDir string) { case CrdPackage: items, err := res.client.PackageList(metav1.NamespaceAll) if err != nil { - log.Printf("Error getting %v list: %v", res.crdType, err) + log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err)) return } @@ -89,7 +90,7 @@ func (res CrdDumper) Dump(dumpDir string) { case CrdHttpTrigger: items, err := res.client.HTTPTriggerList(metav1.NamespaceAll) if err != nil { - log.Printf("Error getting %v list: %v", res.crdType, err) + log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err)) return } @@ -101,7 +102,7 @@ func (res CrdDumper) Dump(dumpDir string) { case CrdKubeWatcher: items, err := res.client.WatchList(metav1.NamespaceAll) if err != nil { - log.Printf("Error getting %v list: %v", res.crdType, err) + log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err)) return } @@ -116,7 +117,7 @@ func (res CrdDumper) Dump(dumpDir string) { for _, mqType := range []string{fission.MessageQueueTypeNats, fission.MessageQueueTypeASQ} { l, err := res.client.MessageQueueTriggerList(mqType, metav1.NamespaceAll) if err != nil { - log.Printf("Error getting %v list: %v", res.crdType, err) + log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err)) break } triggers = append(triggers, l...) @@ -130,7 +131,7 @@ func (res CrdDumper) Dump(dumpDir string) { case CrdTimeTrigger: items, err := res.client.TimeTriggerList(metav1.NamespaceAll) if err != nil { - log.Printf("Error getting %v list: %v", res.crdType, err) + log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err)) return } @@ -140,7 +141,7 @@ func (res CrdDumper) Dump(dumpDir string) { } default: - log.Printf("Unknown type: %v", res.crdType) + log.Info(fmt.Sprintf("Unknown type: %v", res.crdType)) } } diff --git a/fission/support/resources/kubernetes.go b/fission/support/resources/kubernetes.go index 6d76ef18..f3b11172 100644 --- a/fission/support/resources/kubernetes.go +++ b/fission/support/resources/kubernetes.go @@ -21,7 +21,6 @@ import ( "bytes" "fmt" "io" - "log" "path/filepath" "sync" @@ -30,6 +29,7 @@ import ( "k8s.io/client-go/kubernetes" "github.com/fission/fission" + "github.com/fission/fission/fission/log" ) const ( @@ -52,7 +52,7 @@ func NewKubernetesVersion(clientset *kubernetes.Clientset) Resource { func (res KubernetesVersion) Dump(dumpDir string) { serverVer, err := res.client.ServerVersion() if err != nil { - log.Printf("Error setting up kubernetes client: %v", err) + log.Info(fmt.Sprintf("Error setting up kubernetes client: %v", err)) return } @@ -80,7 +80,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) { case KubernetesService: objs, err := res.client.CoreV1().Services(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector}) if err != nil { - log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) + log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)) return } @@ -93,7 +93,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) { case KubernetesDeployment: objs, err := res.client.AppsV1().Deployments(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector}) if err != nil { - log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) + log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)) return } @@ -105,7 +105,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) { case KubernetesPod: objs, err := res.client.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector}) if err != nil { - log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) + log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)) return } @@ -117,7 +117,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) { case KubernetesHPA: objs, err := res.client.AutoscalingV2beta1().HorizontalPodAutoscalers(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector}) if err != nil { - log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) + log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)) return } @@ -129,7 +129,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) { case KubernetesNode: objs, err := res.client.CoreV1().Nodes().List(metav1.ListOptions{LabelSelector: res.selector}) if err != nil { - log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) + log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)) return } @@ -142,7 +142,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) { } default: - log.Printf("Unknown type: %v", res.objType) + log.Info(fmt.Sprintf("Unknown type: %v", res.objType)) } } @@ -187,7 +187,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) { Pods(metav1.NamespaceAll). List(metav1.ListOptions{LabelSelector: res.labelSelector}) if err != nil { - log.Printf("Error getting controller list: %v", err) + log.Info(fmt.Sprintf("Error getting controller list: %v", err)) return } @@ -200,7 +200,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) { defer wg.Done() if !fission.IsReadyPod(&pod) { - fmt.Printf("Pod %v is not in ready state, ignore it\n", pod.Name) + log.Info(fmt.Sprintf("Pod %v is not in ready state, ignore it\n", pod.Name)) return } @@ -211,7 +211,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) { stream, err := req.Stream() if err != nil { - log.Printf("Error streaming logs for pod %v: %v", pod.Name, err) + log.Info(fmt.Sprintf("Error streaming logs for pod %v: %v", pod.Name, err)) return } @@ -225,13 +225,13 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) { stream.Close() break } - log.Printf("Error reading logs from buffer: %v", err) + log.Info(fmt.Sprintf("Error reading logs from buffer: %v", err)) return } _, err = buffer.WriteString(string(line) + "\n") if err != nil { - log.Printf("Error writing bytes to buffer: %v", err) + log.Info(fmt.Sprintf("Error writing bytes to buffer: %v", err)) } } diff --git a/fission/support/resources/resource.go b/fission/support/resources/resource.go index f052354f..4dc70ebc 100644 --- a/fission/support/resources/resource.go +++ b/fission/support/resources/resource.go @@ -19,13 +19,13 @@ package resources import ( "fmt" "io/ioutil" - "log" "path/filepath" "github.com/ghodss/yaml" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" + "github.com/fission/fission/fission/log" ) type Resource interface { @@ -40,7 +40,7 @@ func getFileName(dumpdir string, meta metav1.ObjectMeta) string { func writeToFile(file string, obj interface{}) { bs, err := yaml.Marshal(obj) if err != nil { - log.Printf("Error encoding object: %v", err) + log.Info(fmt.Sprintf("Error encoding object: %v", err)) return } @@ -52,6 +52,6 @@ func writeToFile(file string, obj interface{}) { err = ioutil.WriteFile(file, bs, 0644) if err != nil { - log.Printf("Error writing file %v: %v", file, err) + log.Info(fmt.Sprintf("Error writing file %v: %v", file, err)) } } diff --git a/fission/util/version.go b/fission/util/version.go index 20280516..39ed8b63 100644 --- a/fission/util/version.go +++ b/fission/util/version.go @@ -3,11 +3,12 @@ package util import ( "fmt" + yaml "gopkg.in/yaml.v2" + "github.com/fission/fission" "github.com/fission/fission/controller/client" "github.com/fission/fission/fission/log" "github.com/fission/fission/fission/plugin" - yaml "gopkg.in/yaml.v2" ) // Versions is a container of versions of the client (and its plugins) and server (and its plugins). diff --git a/glide.lock b/glide.lock index fad8953d..8e3a7a83 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: fbb8e5b2fc64114cf4cd7158a1498f36c580e01b327c14cd35682f999ce13db3 -updated: 2019-01-24T19:14:41.16237463+05:30 +hash: 9c39373a78b1e542a42169b1af8fa644f62690e73a988a01bd5ad45a3639d872 +updated: 2019-03-04T16:38:40.987945-07:00 imports: - name: cloud.google.com/go version: 3b1ae45394a234c385be014e9a488f2bb6eef821 @@ -110,12 +110,10 @@ imports: - openstack/identity/v3/tokens - openstack/utils - pagination -- name: github.com/gorilla/context - version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42 - name: github.com/gorilla/handlers version: 7e0847f9db758cdebd26c149d0ae9d5d0b9c98ce - name: github.com/gorilla/mux - version: e3702bed27f0d39777b0b37b664b6280e8ef8fbf + version: a7962380ca08b5a188038c69871b8d3fbdf31e89 - name: github.com/graymeta/stow version: 7b5498c561bbfeb17e413eb96c9edaf88af3855f subpackages: @@ -162,7 +160,7 @@ imports: subpackages: - pb - name: github.com/nats-io/nats-streaming-server - version: 7b758bb93407505bc1d0cf26d091fc281d99ca0c + version: 247d7fa2c948ab918d7b4a9f761010082e8b92ea subpackages: - spb - util @@ -203,7 +201,7 @@ imports: - name: github.com/satori/go.uuid version: f58768cc1a7a7e77a3bd49e98cdd21419399b6a3 - name: github.com/Shopify/sarama - version: 03a43f93cd29dc549e6d9b11892795c206f9c38c + version: 4602b5a8c6e826f9e0737865818dd43b2339a092 - name: github.com/sirupsen/logrus version: 68cec9f21fbf3ea8d8f98c044bc6ce05f17b267a - name: github.com/spf13/pflag @@ -223,7 +221,7 @@ imports: - name: github.com/urfave/cli version: cfb38830724cc34fedffe9a2a29fb54fa9169cd1 - name: go.opencensus.io - version: 2b5032d79456124f42db6b7eb19ac6c155449dc2 + version: f305e5c4e2cf345eba88de13d10de1126fa45a61 subpackages: - exemplar - exporter/jaeger @@ -240,6 +238,18 @@ imports: - trace/internal - trace/propagation - trace/tracestate +- name: go.uber.org/atomic + version: 1ea20fb1cbb1cc08cbd0d913a96dead89aa18289 +- name: go.uber.org/multierr + version: 3c4937480c32f4c13a875a1829af76c98ca3d40a +- name: go.uber.org/zap + version: ff33455a0e382e8a81d14dd7c922020b6b5e7982 + subpackages: + - buffer + - internal/bufferpool + - internal/color + - internal/exit + - zapcore - name: golang.org/x/crypto version: 81e90905daefcd6fd217b62423c0908922eadb30 subpackages: @@ -307,7 +317,7 @@ imports: - name: gopkg.in/yaml.v2 version: 670d4cfef0544295bc27a114dbac37980d83185a - name: k8s.io/api - version: 0f11257a8a25954878633ebdc9841c67d8f83bdb + version: c89978d5f86d7427bef2fc7752732c8c60b1d188 subpackages: - admissionregistration/v1alpha1 - admissionregistration/v1beta1 @@ -338,7 +348,7 @@ imports: - storage/v1alpha1 - storage/v1beta1 - name: k8s.io/apiextensions-apiserver - version: f584b16eb23bd2a3fd292a027d698d95db427c5d + version: f0729a5940c5414a9b9da2f18ba8d5c4b559ec9a subpackages: - pkg/apis/apiextensions - pkg/apis/apiextensions/v1beta1 @@ -346,7 +356,7 @@ imports: - pkg/client/clientset/clientset/scheme - pkg/client/clientset/clientset/typed/apiextensions/v1beta1 - name: k8s.io/apimachinery - version: e386b2658ed20923da8cc9250e552f082899a1ee + version: d49e237a2683fa6dc43a86c7b1b766e0219fb6e7 subpackages: - pkg/api/errors - pkg/api/meta diff --git a/glide.yaml b/glide.yaml index 6c096cd3..05960668 100644 --- a/glide.yaml +++ b/glide.yaml @@ -20,7 +20,9 @@ import: version: ^1.1.0 - package: github.com/urfave/cli version: ^1.18.1 -- package: golang.org/x/tools/imports +- package: golang.org/x/tools + subpackages: + - imports - package: golang.org/x/net subpackages: - context @@ -68,12 +70,18 @@ import: - package: github.com/prometheus/client_golang version: v0.8.0 - package: github.com/dustin/go-humanize -- package: github.com/golang/protobuf/proto +- package: github.com/golang/protobuf version: v1.1.0 + subpackages: + - proto - package: github.com/bsm/sarama-cluster version: ^2.1.11 - package: github.com/Shopify/sarama version: ^1.15.0 - package: go.opencensus.io version: ^0.19.0 -- package: google.golang.org/api/support/bundler \ No newline at end of file +- package: google.golang.org/api + subpackages: + - support/bundler +- package: go.uber.org/zap + version: v1.9.1 diff --git a/kubewatcher/kubewatcher.go b/kubewatcher/kubewatcher.go index 8f322bd6..2f65c207 100644 --- a/kubewatcher/kubewatcher.go +++ b/kubewatcher/kubewatcher.go @@ -21,12 +21,12 @@ import ( "encoding/json" "fmt" "io" - "log" "reflect" "strings" "sync/atomic" "time" + "go.uber.org/zap" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -48,6 +48,7 @@ const ( type ( KubeWatcher struct { + logger *zap.Logger watches map[types.UID]watchSubscription kubernetesClient *kubernetes.Clientset requestChannel chan *kubeWatcherRequest @@ -56,6 +57,7 @@ type ( } watchSubscription struct { + logger *zap.Logger watch crd.KubernetesWatchTrigger kubeWatch watch.Interface lastResourceVersion string @@ -74,8 +76,9 @@ type ( } ) -func MakeKubeWatcher(kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { +func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { kw := &KubeWatcher{ + logger: logger.Named("kube_watcher"), watches: make(map[types.UID]watchSubscription), kubernetesClient: kubernetesClient, publisher: publisher, @@ -167,16 +170,14 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *crd.KubernetesWa case "JOB": wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(listOptions) default: - msg := fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type) - log.Println(msg) - err = errors.NewBadRequest(msg) + err = errors.NewBadRequest(fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type)) } return wi, err } func (kw *KubeWatcher) addWatch(w *crd.KubernetesWatchTrigger) error { - log.Printf("Adding watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference) - ws, err := MakeWatchSubscription(w, kw.kubernetesClient, kw.publisher) + kw.logger.Info("adding watch", zap.String("name", w.Metadata.Name), zap.Any("function", w.Spec.FunctionReference)) + ws, err := MakeWatchSubscription(kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher) if err != nil { return err } @@ -185,7 +186,7 @@ func (kw *KubeWatcher) addWatch(w *crd.KubernetesWatchTrigger) error { } func (kw *KubeWatcher) removeWatch(w *crd.KubernetesWatchTrigger) error { - log.Printf("Removing watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference) + kw.logger.Info("removing watch", zap.String("name", w.Metadata.Name), zap.Any("function", w.Spec.FunctionReference)) ws, ok := kw.watches[w.Metadata.UID] if !ok { return fission.MakeError(fission.ErrorNotFound, @@ -211,9 +212,10 @@ func (kw *KubeWatcher) removeWatch(w *crd.KubernetesWatchTrigger) error { // return nil // } -func MakeWatchSubscription(w *crd.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { +func MakeWatchSubscription(logger *zap.Logger, w *crd.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { var stopped int32 = 0 ws := &watchSubscription{ + logger: logger.Named("watch_subscription"), watch: *w, kubeWatch: nil, stopped: &stopped, @@ -234,8 +236,11 @@ func MakeWatchSubscription(w *crd.KubernetesWatchTrigger, kubeClient *kubernetes func (ws *watchSubscription) restartWatch() error { retries := 60 for { - log.Printf("(re)starting watch %v (ns:%v type:%v) at rv:%v", - ws.watch.Metadata, ws.watch.Spec.Namespace, ws.watch.Spec.Type, ws.lastResourceVersion) + ws.logger.Info("(re)starting watch", + zap.Any("watch", ws.watch.Metadata), + zap.String("namespace", ws.watch.Spec.Namespace), + zap.String("type", ws.watch.Spec.Type), + zap.String("last_resource_version", ws.lastResourceVersion)) wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) if err != nil { retries-- @@ -260,7 +265,7 @@ func getResourceVersion(obj runtime.Object) (string, error) { } func (ws *watchSubscription) eventDispatchLoop() { - log.Println("Listening to watch ", ws.watch.Metadata.Name) + ws.logger.Info("listening to watch", zap.String("name", ws.watch.Metadata.Name)) for { // check watchSubscription is stopped or not before waiting for event // comes from the kubeWatch.ResultChan(). This fix the edge case that @@ -273,14 +278,14 @@ func (ws *watchSubscription) eventDispatchLoop() { if !more { if ws.isStopped() { // watch is removed by user. - log.Println("Watch stopped", ws.watch.Metadata.Name) + ws.logger.Info("watch stopped", zap.String("watch_name", ws.watch.Metadata.Name)) return } else { // watch closed due to timeout, restart it. - log.Printf("Watch %v timed out, restarting", ws.watch.Metadata.Name) + ws.logger.Info("watch timed out - restarting", zap.String("watch_name", ws.watch.Metadata.Name)) err := ws.restartWatch() if err != nil { - log.Panicf("Failed to restart watch: %v", err) + ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name)) } continue } @@ -288,19 +293,19 @@ func (ws *watchSubscription) eventDispatchLoop() { if ev.Type == watch.Error { e := errors.FromObject(ev.Object) - log.Printf("Watch error, retrying in a second: %v", e) + ws.logger.Info("watch error - retrying after one second", zap.Error(e), zap.String("watch_name", ws.watch.Metadata.Name)) // Start from the beginning to get around "too old resource version" ws.lastResourceVersion = "" time.Sleep(time.Second) err := ws.restartWatch() if err != nil { - log.Panicf("Failed to restart watch: %v", err) + ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name)) } continue } rv, err := getResourceVersion(ev.Object) if err != nil { - log.Printf("Error getting resourceVersion from object: %v", err) + ws.logger.Error("error getting resourceVersion from object", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name)) } else { ws.lastResourceVersion = rv } @@ -309,7 +314,7 @@ func (ws *watchSubscription) eventDispatchLoop() { var buf bytes.Buffer err = printKubernetesObject(ev.Object, &buf) if err != nil { - log.Printf("Failed to serialize object: %v", err) + ws.logger.Error("failed to serialize object", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name)) // TODO send a POST request indicating error } @@ -322,8 +327,9 @@ func (ws *watchSubscription) eventDispatchLoop() { // TODO support other function ref types. Or perhaps delegate to router? if ws.watch.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { - log.Printf("Error: unsupported function ref type: %v, can't publish event", - ws.watch.Spec.FunctionReference.Type) + ws.logger.Error("unsupported function ref type - cannot publish event", + zap.Any("type", ws.watch.Spec.FunctionReference.Type), + zap.String("watch_name", ws.watch.Metadata.Name)) continue } diff --git a/kubewatcher/main.go b/kubewatcher/main.go index acad8dcd..ebca054f 100644 --- a/kubewatcher/main.go +++ b/kubewatcher/main.go @@ -17,26 +17,27 @@ limitations under the License. package kubewatcher import ( - "log" + "github.com/pkg/errors" + "go.uber.org/zap" "github.com/fission/fission/crd" "github.com/fission/fission/publisher" ) -func Start(routerUrl string) error { +func Start(logger *zap.Logger, routerUrl string) error { fissionClient, kubeClient, _, err := crd.MakeFissionClient() if err != nil { - return err + return errors.Wrap(err, "failed to get fission or kubernetes client") } err = fissionClient.WaitForCRDs() if err != nil { - log.Fatalf("Error waiting for CRDs: %v", err) + return errors.Wrap(err, "error waiting for CRDs") } - poster := publisher.MakeWebhookPublisher(routerUrl) - kubeWatch := MakeKubeWatcher(kubeClient, poster) - MakeWatchSync(fissionClient, kubeWatch) + poster := publisher.MakeWebhookPublisher(logger, routerUrl) + kubeWatch := MakeKubeWatcher(logger, kubeClient, poster) + MakeWatchSync(logger, fissionClient, kubeWatch) return nil } diff --git a/kubewatcher/watchSync.go b/kubewatcher/watchSync.go index 14fb9ed3..9f0f4ba7 100644 --- a/kubewatcher/watchSync.go +++ b/kubewatcher/watchSync.go @@ -17,9 +17,9 @@ limitations under the License. package kubewatcher import ( - "log" "time" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission/crd" @@ -27,13 +27,15 @@ import ( type ( WatchSync struct { + logger *zap.Logger client *crd.FissionClient kubeWatcher *KubeWatcher } ) -func MakeWatchSync(client *crd.FissionClient, kubeWatcher *KubeWatcher) *WatchSync { +func MakeWatchSync(logger *zap.Logger, client *crd.FissionClient, kubeWatcher *KubeWatcher) *WatchSync { ws := &WatchSync{ + logger: logger.Named("watch_sync"), client: client, kubeWatcher: kubeWatcher, } @@ -46,7 +48,7 @@ func (ws *WatchSync) syncSvc() { for { watches, err := ws.client.KubernetesWatchTriggers(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { - log.Fatalf("Failed to get Kubernetes watch trigger list: %v", err) + ws.logger.Fatal("failed to get Kubernetes watch trigger list", zap.Error(err)) } ws.kubeWatcher.Sync(watches.Items) diff --git a/mqtrigger/main.go b/mqtrigger/main.go index 3f888ecb..cc8f75d1 100644 --- a/mqtrigger/main.go +++ b/mqtrigger/main.go @@ -17,22 +17,24 @@ limitations under the License. package messagequeue import ( - "log" "os" + "github.com/pkg/errors" + "go.uber.org/zap" + "github.com/fission/fission/crd" "github.com/fission/fission/mqtrigger/messageQueue" ) -func Start(routerUrl string) error { +func Start(logger *zap.Logger, routerUrl string) error { fissionClient, _, _, err := crd.MakeFissionClient() if err != nil { - log.Fatalf("Failed to get fission client: %v", err) + return errors.Wrap(err, "failed to get fission or kubernetes client") } err = fissionClient.WaitForCRDs() if err != nil { - log.Fatalf("Error waiting for CRDs: %v", err) + return errors.Wrap(err, "error waiting for CRDs") } // Message queue type: nats is the only supported one for now @@ -42,6 +44,6 @@ func Start(routerUrl string) error { MQType: mqType, Url: mqUrl, } - messageQueue.MakeMessageQueueTriggerManager(fissionClient, routerUrl, mqCfg) + messageQueue.MakeMessageQueueTriggerManager(logger, fissionClient, routerUrl, mqCfg) return nil } diff --git a/mqtrigger/messageQueue/asq.go b/mqtrigger/messageQueue/asq.go index d5c54acf..1a574a62 100644 --- a/mqtrigger/messageQueue/asq.go +++ b/mqtrigger/messageQueue/asq.go @@ -19,7 +19,6 @@ package messageQueue import ( "bytes" "encoding/base64" - "errors" "fmt" "io/ioutil" "net/http" @@ -29,12 +28,12 @@ import ( "sync" "time" + "github.com/Azure/azure-sdk-for-go/storage" + "github.com/pkg/errors" + "go.uber.org/zap" + "github.com/fission/fission" "github.com/fission/fission/crd" - - log "github.com/sirupsen/logrus" - - "github.com/Azure/azure-sdk-for-go/storage" ) // TODO: some of these constants should probably be environment variables @@ -55,6 +54,7 @@ const ( // AzureStorageConnection represents an Azure storage connection. type AzureStorageConnection struct { + logger *zap.Logger routerURL string service AzureQueueService httpClient AzureHTTPClient @@ -173,7 +173,7 @@ func newAzureQueueService(client storage.Client) AzureQueueService { } } -func newAzureStorageConnection(routerURL string, config MessageQueueConfig) (MessageQueue, error) { +func newAzureStorageConnection(logger *zap.Logger, routerURL string, config MessageQueueConfig) (MessageQueue, error) { account := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME") if len(account) == 0 { return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_NAME' is not set") @@ -184,13 +184,14 @@ func newAzureStorageConnection(routerURL string, config MessageQueueConfig) (Mes return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_KEY' is not set") } - log.Infof("Creating Azure storage connection to storage account '%s'.", account) + logger.Info("creating Azure storage connection to storage account", zap.String("account", account)) client, err := storage.NewBasicClient(account, key) if err != nil { - return nil, fmt.Errorf("Failed to Azure create storage client: %v", err) + return nil, errors.Wrap(err, "failed to create Azure storage client") } return &AzureStorageConnection{ + logger: logger.Named("azue_storage"), routerURL: routerURL, service: newAzureQueueService(client), httpClient: &http.Client{ @@ -200,10 +201,10 @@ func newAzureStorageConnection(routerURL string, config MessageQueueConfig) (Mes } func (asc AzureStorageConnection) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscription, error) { - log.Infof("Subscribing to Azure storage queue '%s'.", trigger.Spec.Topic) + asc.logger.Info("subscribing to Azure storage queue", zap.String("queue", trigger.Spec.Topic)) if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { - return nil, fmt.Errorf("Unsupported function reference type (%v) for trigger %v", trigger.Spec.FunctionReference.Type, trigger.Metadata.Name) + return nil, fmt.Errorf("unsupported function reference type (%v) for trigger %q", trigger.Spec.FunctionReference.Type, trigger.Metadata.Name) } subscription := &AzureQueueSubscription{ @@ -226,7 +227,7 @@ func (asc AzureStorageConnection) subscribe(trigger *crd.MessageQueueTrigger) (m func (asc AzureStorageConnection) unsubscribe(subscription messageQueueSubscription) error { sub := subscription.(*AzureQueueSubscription) - log.Infof("Unsubscribing from Azure storage queue '%s'.", sub.queueName) + asc.logger.Info("unsubscribing from Azure storage queue", zap.String("queue", sub.queueName)) // Let the worker know we've unsubscribed sub.unsubscribe <- true @@ -245,7 +246,7 @@ func runAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubsc timer := time.NewTimer(AzureQueuePollingInterval) for { - log.Infof("Waiting for %v before polling Azure storage queue '%s'.", AzureQueuePollingInterval, sub.queueName) + conn.logger.Info("waiting before polling Azure storage queue", zap.Duration("interval_length", AzureQueuePollingInterval), zap.String("queue", sub.queueName)) select { case <-sub.unsubscribe: timer.Stop() @@ -261,18 +262,18 @@ func runAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubsc } func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubscription, wg *sync.WaitGroup) { - log.Infof("Polling messages for Azure storage queue '%s'.", sub.queueName) + conn.logger.Info("polling for messages from Azure storage queue", zap.String("queue", sub.queueName)) err := sub.queue.Create(nil) if err != nil { - log.Errorf("Failed to create message queue '%s': %v", sub.queueName, err) + conn.logger.Error("failed to create message queue", zap.Error(err), zap.String("queue", sub.queueName)) return } for { err := sub.queue.Create(nil) if err != nil { - log.Errorf("Failed to create message queue '%s': %v", sub.queueName, err) + conn.logger.Error("failed to create message queue", zap.Error(err), zap.String("queue", sub.queueName)) return } @@ -281,7 +282,7 @@ func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubs VisibilityTimeout: int(AzureMessageVisibilityTimeout / time.Second), }) if err != nil { - log.Errorf("Failed to retrieve messages from Azure storage queue '%s': %v", sub.queueName, err) + conn.logger.Error("failed to retrieve messages from Azure storage queue", zap.Error(err), zap.String("queue", sub.queueName)) break } if len(messages) == 0 { @@ -301,15 +302,15 @@ func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubs func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscription, message AzureMessage) { defer message.Delete(nil) - log.Printf("Making HTTP request to %s.", sub.functionURL) + conn.logger.Info("making HTTP request to invoke function", zap.String("function_url", sub.functionURL)) for i := 0; i <= AzureQueueRetryLimit; i++ { if i > 0 { - log.Infof("Retry #%d for request to %s.", i, sub.functionURL) + conn.logger.Info("retrying function invocation", zap.Int("retry", i), zap.String("function_url", sub.functionURL)) } request, err := http.NewRequest("POST", sub.functionURL, bytes.NewReader(message.Bytes())) if err != nil { - log.Errorf("Failed to create HTTP request to %s: %v", sub.functionURL, err) + conn.logger.Error("failed to create HTTP request to invoke function", zap.Error(err), zap.String("function_url", sub.functionURL)) continue } @@ -324,19 +325,22 @@ func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscri response, err := conn.httpClient.Do(request) if err != nil { - log.Errorf("Request to %s failed: %v", sub.functionURL, err) + conn.logger.Error("sending function invocation request failed", zap.Error(err), zap.String("function_url", sub.functionURL)) continue } defer response.Body.Close() body, err := ioutil.ReadAll(response.Body) if err != nil { - log.Errorf("Failed to read response body from %s: %v.", sub.functionURL, err) + conn.logger.Error("failed to read response body from function invocation", zap.Error(err), zap.String("function_url", sub.functionURL)) continue } if response.StatusCode < 200 || response.StatusCode >= 300 { - log.Printf("Request to %s returned failure: %s (%d).", sub.functionURL, string(body), response.StatusCode) + conn.logger.Error("function invocation request returned a failure status code", + zap.String("function_url", sub.functionURL), + zap.String("body", string(body)), + zap.Int("status_code", response.StatusCode)) continue } @@ -344,14 +348,19 @@ func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscri outputQueue := conn.service.GetQueue(sub.outputQueueName) err = outputQueue.Create(nil) if err != nil { - log.Errorf("Failed to create output queue '%s': %v.", sub.outputQueueName, err) + conn.logger.Error("failed to create output queue", + zap.Error(err), + zap.String("output_queue", sub.outputQueueName), + zap.String("function_url", sub.functionURL)) return } outputMessage := outputQueue.NewMessage(string(body)) err = outputMessage.Put(nil) if err != nil { - log.Errorf("Failed to post response body from %s to output queue '%s': %v.", sub.functionURL, sub.outputQueueName, err) + conn.logger.Error("failed to post response body from function invocation to output queue", + zap.String("output_queue", sub.outputQueueName), + zap.String("function_url", sub.functionURL)) return } } @@ -360,20 +369,28 @@ func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscri return } - log.Errorf("Request to %s failed after %d retries; moving message to poison queue.", sub.functionURL, AzureQueueRetryLimit) + conn.logger.Error("function invocation retired too many times - moving message to poison queue", + zap.Int("retry_limit", AzureQueueRetryLimit), + zap.String("function_url", sub.functionURL)) poisonQueueName := sub.queueName + AzurePoisonQueueSuffix poisonQueue := conn.service.GetQueue(poisonQueueName) err := poisonQueue.Create(nil) if err != nil { - log.Errorf("Failed to create poison queue '%s': %v", poisonQueueName, err) + conn.logger.Error("failed to create poison queue", + zap.Error(err), + zap.String("poison_queue_name", poisonQueueName), + zap.String("function_url", sub.functionURL)) return } poisonMessage := poisonQueue.NewMessage(string(message.Bytes())) err = poisonMessage.Put(nil) if err != nil { - log.Errorf("Failed to post response body from %s to output queue '%s': %v", sub.functionURL, poisonQueueName, err) + conn.logger.Error("failed to post response body from function invocation failure poison queue", + zap.Error(err), + zap.String("poison_queue_name", poisonQueueName), + zap.String("function_url", sub.functionURL)) return } } diff --git a/mqtrigger/messageQueue/asq_test.go b/mqtrigger/messageQueue/asq_test.go index 131bef9e..655fc80d 100644 --- a/mqtrigger/messageQueue/asq_test.go +++ b/mqtrigger/messageQueue/asq_test.go @@ -18,26 +18,33 @@ package messageQueue import ( "fmt" "io/ioutil" + "log" "net/http" "os" "strings" "testing" "time" - "github.com/fission/fission" - "github.com/fission/fission/crd" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/Azure/azure-sdk-for-go/storage" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/fission/fission" + "github.com/fission/fission/crd" ) const ( DummyRouterURL = "http://localhost" ) +func panicIf(err error) { + if err != nil { + log.Panicf("Error: %v", err) + } +} + type azureQueueServiceMock struct { mock.Mock } @@ -103,7 +110,10 @@ func (m *azureHTTPClientMock) Do(req *http.Request) (*http.Response, error) { } func TestNewStorageConnectionMissingAccountName(t *testing.T) { - connection, err := newAzureStorageConnection(DummyRouterURL, MessageQueueConfig{ + logger, err := zap.NewDevelopment() + panicIf(err) + + connection, err := newAzureStorageConnection(logger, DummyRouterURL, MessageQueueConfig{ MQType: fission.MessageQueueTypeASQ, Url: "", }) @@ -112,8 +122,11 @@ func TestNewStorageConnectionMissingAccountName(t *testing.T) { } func TestNewStorageConnectionMissingAccessKey(t *testing.T) { + logger, err := zap.NewDevelopment() + panicIf(err) + _ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname") - connection, err := newAzureStorageConnection(DummyRouterURL, MessageQueueConfig{ + connection, err := newAzureStorageConnection(logger, DummyRouterURL, MessageQueueConfig{ MQType: fission.MessageQueueTypeASQ, Url: "", }) @@ -123,9 +136,12 @@ func TestNewStorageConnectionMissingAccessKey(t *testing.T) { } func TestNewStorageConnection(t *testing.T) { + logger, err := zap.NewDevelopment() + panicIf(err) + _ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname") _ = os.Setenv("AZURE_STORAGE_ACCOUNT_KEY", "bm90IGEga2V5") - connection, err := newAzureStorageConnection(DummyRouterURL, MessageQueueConfig{ + connection, err := newAzureStorageConnection(logger, DummyRouterURL, MessageQueueConfig{ MQType: "azure-storage-queue", Url: "", }) @@ -276,8 +292,12 @@ func TestAzureStorageQueuePoisonMessage(t *testing.T) { service.On("GetQueue", QueueName).Return(queue).Once() service.On("GetQueue", QueueName+AzurePoisonQueueSuffix).Return(poisonQueue).Once() + logger, err := zap.NewDevelopment() + panicIf(err) + // Create the storage connection and subscribe to the trigger connection := AzureStorageConnection{ + logger: logger, routerURL: DummyRouterURL, service: service, httpClient: httpClient, @@ -420,8 +440,12 @@ func runAzureStorageQueueTest(t *testing.T, count int, output bool) { service.On("GetQueue", OutputQueueName).Return(outputQueue).Times(count) } + logger, err := zap.NewDevelopment() + panicIf(err) + // Create the storage connection and subscribe to the trigger connection := AzureStorageConnection{ + logger: logger, routerURL: DummyRouterURL, service: service, httpClient: httpClient, diff --git a/mqtrigger/messageQueue/kafka.go b/mqtrigger/messageQueue/kafka.go index 204498ed..cc8012e8 100644 --- a/mqtrigger/messageQueue/kafka.go +++ b/mqtrigger/messageQueue/kafka.go @@ -26,7 +26,7 @@ import ( sarama "github.com/Shopify/sarama" cluster "github.com/bsm/sarama-cluster" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" "github.com/fission/fission" "github.com/fission/fission/crd" @@ -34,30 +34,35 @@ import ( type ( Kafka struct { + logger *zap.Logger routerUrl string brokers []string version sarama.KafkaVersion } ) -func makeKafkaMessageQueue(routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) { +func makeKafkaMessageQueue(logger *zap.Logger, routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) { if len(routerUrl) == 0 || len(mqCfg.Url) == 0 { - return nil, errors.New("The router URL or MQ URL is empty") + return nil, errors.New("the router URL or MQ URL is empty") } mqKafkaVersion := os.Getenv("MESSAGE_QUEUE_KAFKA_VERSION") // Parse version string kafkaVersion, err := sarama.ParseKafkaVersion(mqKafkaVersion) if err != nil { - log.Warningf("Error parsing version string %q: %v. Falling back to %q", mqKafkaVersion, err, kafkaVersion) + logger.Warn("error parsing kafka version string - falling back to default", + zap.Error(err), + zap.String("failed_version", mqKafkaVersion), + zap.Any("default_version", kafkaVersion)) } kafka := Kafka{ + logger: logger.Named("kafka"), routerUrl: routerUrl, brokers: strings.Split(mqCfg.Url, ","), version: kafkaVersion, } - log.Infof("Created Queue %v", kafka) + logger.Info("created kafka queue", zap.Any("kafka", kafka)) return kafka, nil } @@ -66,8 +71,8 @@ func isTopicValidForKafka(topic string) bool { } func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscription, error) { - log.Infof("Inside kakfa subscribe %q", trigger) - log.Infof("brokers set to %q", kafka.brokers) + kafka.logger.Info("inside kakfa subscribe", zap.Any("trigger", trigger)) + kafka.logger.Info("brokers set", zap.Strings("brokers", kafka.brokers)) // Create new consumer consumerConfig := cluster.NewConfig() @@ -75,7 +80,7 @@ func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubs consumerConfig.Group.Return.Notifications = true consumerConfig.Config.Version = kafka.version consumer, err := cluster.NewConsumer(kafka.brokers, string(trigger.Metadata.UID), []string{trigger.Spec.Topic}, consumerConfig) - log.Infof("Created a new consumer: %#v", consumer) + kafka.logger.Info("created a new consumer", zap.Any("consumer", consumer)) if err != nil { panic(err) } @@ -87,7 +92,7 @@ func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubs producerConfig.Producer.Return.Successes = true producerConfig.Version = kafka.version producer, err := sarama.NewSyncProducer(kafka.brokers, producerConfig) - log.Infof("Created a new producer %q", producer) + kafka.logger.Info("created a new producer", zap.Any("consumer", producer)) if err != nil { panic(err) } @@ -95,21 +100,21 @@ func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubs // consume errors go func() { for err := range consumer.Errors() { - log.Printf("Error: %s\n", err.Error()) + kafka.logger.Error("consumer error", zap.Error(err)) } }() // consume notifications go func() { for ntf := range consumer.Notifications() { - log.Printf("Rebalanced: %+v\n", ntf) + kafka.logger.Info("consumer notification", zap.Any("notification", ntf)) } }() // consume messages go func() { for msg := range consumer.Messages() { - log.Infof("Calling message handler with value " + string(msg.Value[:])) + kafka.logger.Info("calling message handler", zap.String("message", string(msg.Value[:]))) if kafkaMsgHandler(&kafka, producer, trigger, msg) { consumer.MarkOffset(msg, "") // mark message as processed } @@ -127,12 +132,13 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me var value string = string(msg.Value[:]) // Support other function ref types if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { - log.Fatalf("Unsupported function reference type (%v) for trigger %v", - trigger.Spec.FunctionReference.Type, trigger.Metadata.Name) + kafka.logger.Fatal("unsupported function reference type for trigger", + zap.Any("function_reference_type", trigger.Spec.FunctionReference.Type), + zap.String("trigger", trigger.Metadata.Name)) } url := kafka.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name, trigger.Metadata.Namespace), "/") - log.Printf("Making HTTP request to %v", url) + kafka.logger.Info("making HTTP request", zap.String("url", url)) // Generate the Headers fissionHeaders := map[string]string{ @@ -145,7 +151,9 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me // Create request req, err := http.NewRequest("POST", url, strings.NewReader(value)) if err != nil { - log.Warningf("Request creation failed: %v", url) + kafka.logger.Error("failed to create HTTP request to invoke function", + zap.Error(err), + zap.String("function_url", url)) return false } @@ -156,7 +164,8 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me req.Header.Add(string(h.Key), string(h.Value)) } } else { - log.Warningf("Headers are not supported by Kafka version %q, needs v0.11+: no record headers to add in HTTP request", kafka.version) + kafka.logger.Warn("headers are not supported by current Kafka version, needs v0.11+: no record headers to add in HTTP request", + zap.Any("current_version", kafka.version)) } for k, v := range fissionHeaders { @@ -169,7 +178,10 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me // Make the request resp, err = http.DefaultClient.Do(req) if err != nil { - log.Errorf("Error invoking function for trigger %v: %v", trigger.Metadata.Name, err) + kafka.logger.Error("sending function invocation request failed", + zap.Error(err), + zap.String("function_url", url), + zap.String("trigger", trigger.Metadata.Name)) continue } if resp == nil { @@ -182,18 +194,23 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me } if resp == nil { - log.Warning("Every retry failed; final retry gave empty response.") + kafka.logger.Warn("every function invocation retry failed; final retry gave empty response", + zap.String("function_url", url), + zap.String("trigger", trigger.Metadata.Name)) return false } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) - log.Infof("Got response " + string(body)) + kafka.logger.Info("got response from function invocation", + zap.String("function_url", url), + zap.String("trigger", trigger.Metadata.Name), + zap.String("body", string(body))) if err != nil { - errorHandler(trigger, producer, fmt.Sprintf("Request body error: %v", string(body))) + errorHandler(kafka.logger, trigger, producer, fmt.Sprintf("request body error: %v", string(body))) return false } if resp.StatusCode != 200 { - errorHandler(trigger, producer, fmt.Sprintf("Request returned failure: %v", resp.StatusCode)) + errorHandler(kafka.logger, trigger, producer, fmt.Sprintf("request returned failure: %v", resp.StatusCode)) return false } if len(trigger.Spec.ResponseTopic) > 0 { @@ -207,7 +224,8 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me } } } else { - log.Warningf("Headers are not supported by Kafka version %q, needs v0.11+: dropping the headers", kafka.version) + kafka.logger.Warn("headers are not supported by current Kafka version, needs v0.11+: no record headers to add in HTTP request", + zap.Any("current_version", kafka.version)) } _, _, err := producer.SendMessage(&sarama.ProducerMessage{ @@ -216,24 +234,30 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me Headers: kafkaRecordHeaders, }) if err != nil { - log.Warningf("Failed to publish message to topic %s: %v", trigger.Spec.ResponseTopic, err) + kafka.logger.Warn("failed to publish response body from function invocation to topic", + zap.Error(err), + zap.String("topic", trigger.Spec.Topic), + zap.String("function_url", url)) return false } } return true } -func errorHandler(trigger *crd.MessageQueueTrigger, producer sarama.SyncProducer, body string) { +func errorHandler(logger *zap.Logger, trigger *crd.MessageQueueTrigger, producer sarama.SyncProducer, body string) { if len(trigger.Spec.ErrorTopic) > 0 { _, _, err := producer.SendMessage(&sarama.ProducerMessage{ Topic: trigger.Spec.ErrorTopic, Value: sarama.StringEncoder(body), }) if err != nil { - log.Warningf("Failed to publish message to error topic %s: %v", trigger.Spec.ErrorTopic, err) + logger.Warn("failed to publish message to error topic", + zap.Error(err), + zap.String("topic", trigger.Spec.Topic)) return } } else { - log.Printf(body) + logger.Error("message received to publish to error topic, but no error topic was set", + zap.String("message", body)) } } diff --git a/mqtrigger/messageQueue/messageQueue.go b/mqtrigger/messageQueue/messageQueue.go index 0dd254a7..f0d43ea3 100644 --- a/mqtrigger/messageQueue/messageQueue.go +++ b/mqtrigger/messageQueue/messageQueue.go @@ -18,9 +18,10 @@ package messageQueue import ( "errors" + "fmt" "time" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" @@ -50,6 +51,7 @@ type ( } MessageQueueTriggerManager struct { + logger *zap.Logger reqChan chan request mqCfg MessageQueueConfig triggers map[string]*triggerSubscription @@ -73,27 +75,28 @@ type ( } ) -func MakeMessageQueueTriggerManager(fissionClient *crd.FissionClient, routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager { +func MakeMessageQueueTriggerManager(logger *zap.Logger, fissionClient *crd.FissionClient, routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager { var messageQueue MessageQueue var err error mqTriggerMgr := MessageQueueTriggerManager{ + logger: logger.Named("message_queue_trigger_manager"), reqChan: make(chan request), triggers: make(map[string]*triggerSubscription), fissionClient: fissionClient, } switch mqConfig.MQType { case fission.MessageQueueTypeNats: - messageQueue, err = makeNatsMessageQueue(routerUrl, mqConfig) + messageQueue, err = makeNatsMessageQueue(logger, routerUrl, mqConfig) case fission.MessageQueueTypeASQ: - messageQueue, err = newAzureStorageConnection(routerUrl, mqConfig) + messageQueue, err = newAzureStorageConnection(logger, routerUrl, mqConfig) case fission.MessageQueueTypeKafka: - messageQueue, err = makeKafkaMessageQueue(routerUrl, mqConfig) + messageQueue, err = makeKafkaMessageQueue(logger, routerUrl, mqConfig) default: - err = errors.New("No matched message queue type found") + err = fmt.Errorf("no supported message queue type found for %q", mqConfig.MQType) } if err != nil { - log.Fatalf("Failed to connect to remote message queue server: %v", err) + logger.Fatal("failed to connect to remote message queue server", zap.Error(err)) } mqTriggerMgr.messageQueue = messageQueue go mqTriggerMgr.service() @@ -109,7 +112,7 @@ func (mqt *MessageQueueTriggerManager) service() { var err error k := crd.CacheKey(&req.triggerSub.trigger.Metadata) if _, ok := mqt.triggers[k]; ok { - err = errors.New("Trigger already exists") + err = errors.New("trigger already exists") } else { mqt.triggers[k] = req.triggerSub } @@ -164,11 +167,11 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() { newTriggers, err := mqt.fissionClient.MessageQueueTriggers(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { if fission.IsNetworkError(err) { - log.Printf("Encounter network error, retry again: %v", err) + mqt.logger.Info("encountered network error, will retry", zap.Error(err)) time.Sleep(5 * time.Second) continue } - log.Fatalf("Failed to read message queue trigger list: %v", err) + mqt.logger.Fatal("failed to read message queue trigger list", zap.Error(err)) } newTriggerMap := make(map[string]*crd.MessageQueueTrigger) for index := range newTriggers.Items { @@ -188,7 +191,7 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() { // actually subscribe using the message queue client impl sub, err := mqt.messageQueue.subscribe(trigger) if err != nil { - log.Warnf("Failed to subscribe to message queue trigger %s: %v", trigger.Metadata.Name, err) + mqt.logger.Warn("failed to subscribe to message queue trigger", zap.Error(err), zap.String("trigger_name", trigger.Metadata.Name)) continue } @@ -200,10 +203,10 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() { // add to our list err = mqt.addTrigger(&triggerSub) if err != nil { - log.Fatalf("Message queue trigger %s addition failed: %v", trigger.Metadata.Name, err) + mqt.logger.Fatal("adding message queue trigger failed", zap.Error(err), zap.String("trigger_name", trigger.Metadata.Name)) } - log.Infof("Message queue trigger %s created", trigger.Metadata.Name) + mqt.logger.Info("message queue trigger created", zap.String("trigger_name", trigger.Metadata.Name)) } // remove old triggers @@ -213,11 +216,11 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() { } err := mqt.messageQueue.unsubscribe(triggerSub.subscription) if err != nil { - log.Warnf("Failed to unsubscribe to trigger %s: %v", triggerSub.trigger.Metadata.Name, err) + mqt.logger.Warn("failed to unsubscribe from message queue trigger", zap.Error(err), zap.String("trigger_name", triggerSub.trigger.Metadata.Name)) continue } mqt.delTrigger(&triggerSub.trigger.Metadata) - log.Infof("Message queue trigger %s deleted", triggerSub.trigger.Metadata.Name) + mqt.logger.Info("message queue trigger deleted", zap.String("trigger_name", triggerSub.trigger.Metadata.Name)) } // TODO replace with a watch diff --git a/mqtrigger/messageQueue/nats.go b/mqtrigger/messageQueue/nats.go index 2721cc2f..8e4eb129 100644 --- a/mqtrigger/messageQueue/nats.go +++ b/mqtrigger/messageQueue/nats.go @@ -18,7 +18,6 @@ package messageQueue import ( "bytes" - "errors" "fmt" "io/ioutil" "net/http" @@ -26,7 +25,7 @@ import ( ns "github.com/nats-io/go-nats-streaming" nsUtil "github.com/nats-io/nats-streaming-server/util" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" "github.com/fission/fission" "github.com/fission/fission/crd" @@ -41,17 +40,19 @@ const ( type ( Nats struct { + logger *zap.Logger nsConn ns.Conn routerUrl string } ) -func makeNatsMessageQueue(routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) { +func makeNatsMessageQueue(logger *zap.Logger, routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) { conn, err := ns.Connect(natsClusterID, natsClientID, ns.NatsURL(mqCfg.Url)) if err != nil { return nil, err } nats := Nats{ + logger: logger.Named("nats"), nsConn: conn, routerUrl: routerUrl, } @@ -62,7 +63,7 @@ func (nats Nats) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscr subj := trigger.Spec.Topic if !isTopicValidForNats(subj) { - return nil, errors.New(fmt.Sprintf("Not a valid topic: %s", trigger.Spec.Topic)) + return nil, fmt.Errorf("not a valid topic: %q", trigger.Spec.Topic) } opts := []ns.SubscriptionOption{ @@ -96,15 +97,16 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) { // Support other function ref types if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { - log.Fatalf("Unsupported function reference type (%v) for trigger %v", - trigger.Spec.FunctionReference.Type, trigger.Metadata.Name) + nats.logger.Fatal("unsupported function reference type for trigger", + zap.Any("function_reference_type", trigger.Spec.FunctionReference.Type), + zap.String("trigger", trigger.Metadata.Name)) } // with the addition of multi-tenancy, the users can create functions in any namespace. however, // the triggers can only be created in the same namespace as the function. // so essentially, function namespace = trigger namespace. url := nats.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name, trigger.Metadata.Namespace), "/") - log.Printf("Making HTTP request to %v", url) + nats.logger.Info("making HTTP request", zap.String("url", url)) headers := map[string]string{ "X-Fission-MQTrigger-Topic": trigger.Spec.Topic, @@ -117,7 +119,9 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) { req, err := http.NewRequest("POST", url, bytes.NewReader(msg.Data)) if err != nil { - log.Errorf("Could not issue POST request with message to url %v", url) + nats.logger.Error("failed to create HTTP request to invoke function", + zap.Error(err), + zap.String("function_url", url)) return } @@ -130,7 +134,10 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) { // Make the request resp, err = http.DefaultClient.Do(req) if err != nil { - log.Errorf("Error invoking function for trigger %v: %v", trigger.Metadata.Name, err) + nats.logger.Error("sending function invocation request failed", + zap.Error(err), + zap.String("function_url", url), + zap.String("trigger", trigger.Metadata.Name)) continue } if resp == nil { @@ -143,7 +150,9 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) { } if resp == nil { - log.Warning("Every retry failed; final retry gave empty response.") + nats.logger.Warn("every function invocation retry failed; final retry gave empty response", + zap.String("function_url", url), + zap.String("trigger", trigger.Metadata.Name)) return } @@ -151,7 +160,10 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) { body, bodyErr := ioutil.ReadAll(resp.Body) if bodyErr != nil { - log.Warningf("Response body error: %v", bodyErr) + nats.logger.Error("error reading function invocation response", + zap.Error(err), + zap.String("function_url", url), + zap.String("trigger", trigger.Metadata.Name)) return } @@ -160,7 +172,11 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) { if len(trigger.Spec.ErrorTopic) > 0 && len(body) > 0 { publishErr := nats.nsConn.Publish(trigger.Spec.ErrorTopic, body) if publishErr != nil { - log.Errorf("Failed to publish error to error topic: %v", publishErr) + nats.logger.Error("failed to publish function invocation error to error topic", + zap.Error(publishErr), + zap.String("topic", trigger.Spec.ErrorTopic), + zap.String("function_url", url), + zap.String("trigger", trigger.Metadata.Name)) // TODO: We will ack this message after max retries to prevent re-processing but // this may cause message loss } @@ -171,13 +187,19 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) { // Trigger acks message only if a request was processed successfully err = msg.Ack() if err != nil { - log.Warningf("Failed to ack message: %v", err) + nats.logger.Error("failed to ack message after successful function invocation from trigger", + zap.Error(err), + zap.String("function_url", url), + zap.String("trigger", trigger.Metadata.Name)) } if len(trigger.Spec.ResponseTopic) > 0 { err = nats.nsConn.Publish(trigger.Spec.ResponseTopic, body) if err != nil { - log.Warningf("Failed to publish message to topic %s: %v", trigger.Spec.ResponseTopic, err) + nats.logger.Error("failed to publish message with function invocation response to topic", + zap.Error(err), + zap.String("topic", trigger.Spec.ResponseTopic), + zap.String("trigger", trigger.Metadata.Name)) } } } diff --git a/preupgradechecks/main.go b/preupgradechecks/main.go index dd8610d7..39048777 100644 --- a/preupgradechecks/main.go +++ b/preupgradechecks/main.go @@ -17,10 +17,10 @@ limitations under the License. package main import ( - "os" + "log" "github.com/docopt/docopt-go" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" "github.com/fission/fission" ) @@ -34,6 +34,12 @@ func getStringArgWithDefault(arg interface{}, defaultValue string) string { } func main() { + logger, err := zap.NewProduction() + if err != nil { + log.Fatalf("can't initialize zap logger: %v", err) + } + defer logger.Sync() + usage := `Package to perform operations needed prior to fission installation Usage: pre-upgrade-checks --fn-pod-namespace= --envbuilder-namespace= @@ -43,20 +49,20 @@ Options: arguments, err := docopt.Parse(usage, nil, true, fission.BuildInfo().String(), false) if err != nil { - log.Fatalf("Error: %v", err) + logger.Fatal("Could not parse command line arguments", zap.Error(err)) } functionPodNs := getStringArgWithDefault(arguments["--fn-pod-namespace"], "fission-function") envBuilderNs := getStringArgWithDefault(arguments["--envbuilder-namespace"], "fission-builder") - crdBackedClient, err := makePreUpgradeTaskClient(functionPodNs, envBuilderNs) + crdBackedClient, err := makePreUpgradeTaskClient(logger, functionPodNs, envBuilderNs) if err != nil { - log.Printf("Error creating a crd client : %v, please retry helm upgrade", err) - os.Exit(1) + logger.Fatal("error creating a crd client, please retry helm upgrade", + zap.Error(err)) } if !crdBackedClient.IsFissionReInstall() { - log.Printf("Nothing to do since CRDs are not present on the cluster") + logger.Info("nothing to do since CRDs are not present on the cluster") return } diff --git a/preupgradechecks/preupgradechecks.go b/preupgradechecks/preupgradechecks.go index 5a044298..05bfb920 100644 --- a/preupgradechecks/preupgradechecks.go +++ b/preupgradechecks/preupgradechecks.go @@ -18,10 +18,10 @@ package main import ( "fmt" - "os" multierror "github.com/hashicorp/go-multierror" - log "github.com/sirupsen/logrus" + "github.com/pkg/errors" + "go.uber.org/zap" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -33,6 +33,7 @@ import ( type ( PreUpgradeTaskClient struct { + logger *zap.Logger fissionClient *crd.FissionClient k8sClient *kubernetes.Clientset apiExtClient *apiextensionsclient.Clientset @@ -46,19 +47,14 @@ const ( FunctionCRD = "functions.fission.io" ) -func fatal(msg string) { - os.Stderr.WriteString(msg + "\n") - os.Exit(1) -} - -func makePreUpgradeTaskClient(fnPodNs, envBuilderNs string) (*PreUpgradeTaskClient, error) { +func makePreUpgradeTaskClient(logger *zap.Logger, fnPodNs, envBuilderNs string) (*PreUpgradeTaskClient, error) { fissionClient, k8sClient, apiExtClient, err := crd.MakeFissionClient() if err != nil { - log.Errorf("Error making fission client") - return nil, err + return nil, errors.Wrap(err, "error making fission client") } return &PreUpgradeTaskClient{ + logger: logger.Named("pre_upgrade_task_client"), fissionClient: fissionClient, k8sClient: k8sClient, fnPodNs: fnPodNs, @@ -86,7 +82,7 @@ func (client *PreUpgradeTaskClient) IsFissionReInstall() bool { // VerifyFunctionSpecReferences verifies that a function references secrets, configmaps, pkgs in its own namespace and // outputs a list of functions that don't adhere to this requirement. func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() { - log.Printf("Verifying Function spec references for all functions in the cluster") + client.logger.Info("verifying function spec references for all functions in the cluster") var result *multierror.Error var err error @@ -100,7 +96,9 @@ func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() { } if err != nil { - fatal(fmt.Sprintf("Error: %v listing functions even after %d retries", err, MaxRetries)) + client.logger.Fatal("error listing functions after max retries", + zap.Error(err), + zap.Int("max_retries", MaxRetries)) } // check that all secrets, configmaps, packages are in the same namespace @@ -125,12 +123,12 @@ func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() { } if result != nil { - log.Printf("Installation failed due to the following errors :") - log.Printf("Summary : A function cannot reference secrets, configmaps and packages outside it's own namespace") - fatal(result.Error()) + client.logger.Fatal("installation failed", + zap.Error(err), + zap.String("summary", "a function cannot reference secrets, configmaps and packages outside it's own namespace")) } - log.Printf("Function Spec References verified") + client.logger.Info("function spec references verified") } // deleteClusterRoleBinding deletes the clusterRoleBinding passed as an argument to it. @@ -152,11 +150,13 @@ func (client *PreUpgradeTaskClient) RemoveClusterAdminRolesForFissionSAs() { for _, clusterRoleBinding := range clusterRoleBindings { err := client.deleteClusterRoleBinding(clusterRoleBinding) if err != nil { - fatal(fmt.Sprintf("Error deleting rolebinding : %s, err : %v", clusterRoleBinding, err)) + client.logger.Fatal("error deleting rolebinding", + zap.Error(err), + zap.String("role_binding", clusterRoleBinding)) } } - log.Println("Removed cluster admin privileges for fission-builder and fission-fetcher Service Accounts") + client.logger.Info("femoved cluster admin privileges for fission-builder and fission-fetcher service accounts") } // NeedRoleBindings checks if there is atleast one package or function in default namespace. @@ -181,27 +181,40 @@ func (client *PreUpgradeTaskClient) NeedRoleBindings() bool { // Setup appropriate role bindings for fission-fetcher and fission-builder SAs func (client *PreUpgradeTaskClient) SetupRoleBindings() { if !client.NeedRoleBindings() { - log.Printf("No fission objects found, so no role-bindings to create") + client.logger.Info("no fission objects found, so no role-bindings to create") return } // the fact that we're here implies that there had been a prior installation of fission and objects are present still // so, we go ahead and create the role-bindings necessary for the fission-fetcher and fission-builder Service Accounts. - err := fission.SetupRoleBinding(client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs) + err := fission.SetupRoleBinding(client.logger, client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs) if err != nil { - fatal(fmt.Sprintf("Error setting up rolebinding %s for %s.%s service account", fission.PackageGetterRB, fission.FissionFetcherSA, client.fnPodNs)) + client.logger.Fatal("error setting up rolebinding for service account", + zap.Error(err), + zap.String("role_binding", fission.PackageGetterRB), + zap.String("service_account", fission.FissionFetcherSA), + zap.String("service_account_namespace", client.fnPodNs)) } - err = fission.SetupRoleBinding(client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, client.envBuilderNs) + err = fission.SetupRoleBinding(client.logger, client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, client.envBuilderNs) if err != nil { - fatal(fmt.Sprintf("Error setting up rolebinding %s for %s.%s service account", fission.PackageGetterRB, fission.FissionBuilderSA, client.envBuilderNs)) + client.logger.Fatal("error setting up rolebinding for service account", + zap.Error(err), + zap.String("role_binding", fission.PackageGetterRB), + zap.String("service_account", fission.FissionBuilderSA), + zap.String("service_account_namespace", client.envBuilderNs)) } - err = fission.SetupRoleBinding(client.k8sClient, fission.SecretConfigMapGetterRB, metav1.NamespaceDefault, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs) + err = fission.SetupRoleBinding(client.logger, client.k8sClient, fission.SecretConfigMapGetterRB, metav1.NamespaceDefault, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs) if err != nil { - fatal(fmt.Sprintf("Error setting up rolebinding %s for %s.%s service account", fission.SecretConfigMapGetterRB, fission.FissionFetcherSA, client.fnPodNs)) + client.logger.Fatal("error setting up rolebinding for service account", + zap.Error(err), + zap.String("role_binding", fission.SecretConfigMapGetterRB), + zap.String("service_account", fission.FissionFetcherSA), + zap.String("service_account_namespace", client.fnPodNs)) } - log.Printf("Created role-bindings : %s and %s in default namespace", fission.PackageGetterRB, fission.SecretConfigMapGetterRB) + client.logger.Info("created rolebindings in default namespace", + zap.Strings("role_bindings", []string{fission.PackageGetterRB, fission.SecretConfigMapGetterRB})) return } diff --git a/publisher/webhookPublisher.go b/publisher/webhookPublisher.go index 7072aca3..6c30e666 100644 --- a/publisher/webhookPublisher.go +++ b/publisher/webhookPublisher.go @@ -19,15 +19,18 @@ package publisher import ( "bytes" "io/ioutil" - "log" "net/http" "strings" "time" + + "go.uber.org/zap" ) type ( // A webhook publisher for a single URL. Satisifies the Publisher interface. WebhookPublisher struct { + logger *zap.Logger + requestChannel chan *publishRequest maxRetries int @@ -44,8 +47,9 @@ type ( } ) -func MakeWebhookPublisher(baseUrl string) *WebhookPublisher { +func MakeWebhookPublisher(logger *zap.Logger, baseUrl string) *WebhookPublisher { p := &WebhookPublisher{ + logger: logger.Named("webhook_publisher"), baseUrl: baseUrl, requestChannel: make(chan *publishRequest, 32), // buffered channel // TODO make this configurable @@ -76,13 +80,16 @@ func (p *WebhookPublisher) svc() { func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) { url := p.baseUrl + "/" + strings.TrimPrefix(r.target, "/") - log.Printf("Making HTTP request to %v", url) + p.logger.Info("making HTTP request", zap.String("url", url)) var buf bytes.Buffer buf.WriteString(r.body) // Create request req, err := http.NewRequest("POST", url, &buf) + if err != nil { + p.logger.Error("error creating request", zap.Error(err), zap.String("url", url)) + } for k, v := range r.headers { req.Header.Set(k, v) } @@ -98,13 +105,25 @@ func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) { // Log errors if err != nil { - log.Printf("Request failed: %v", r) + p.logger.Error("request failed", + zap.Error(err), + zap.Any("request", r), + zap.String("url", url)) } else if resp.StatusCode != 200 { - log.Printf("Request returned failure: %v", resp.StatusCode) + p.logger.Error("request returned failure status code", + zap.Any("request", r), + zap.String("url", url), + zap.Int("status_code", resp.StatusCode)) body, err := ioutil.ReadAll(resp.Body) resp.Body.Close() - if err == nil { - log.Printf("request error: %v", string(body)) + if err != nil { + p.logger.Error("error reading error request body", + zap.Error(err), + zap.Any("request", r), + zap.String("url", url), + zap.Int("status_code", resp.StatusCode)) + } else { + p.logger.Error("request error", zap.String("body", string(body))) } } @@ -116,7 +135,7 @@ func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) { p.requestChannel <- r }) } else { - log.Printf("Final retry failed, giving up on %v", url) + p.logger.Error("final retry failed, giving up", zap.String("url", url)) // Event dropped } } diff --git a/redis/redis.go b/redis/redis.go index e82883c8..79d5e15b 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -26,30 +26,30 @@ import ( "github.com/golang/protobuf/proto" "github.com/gomodule/redigo/redis" - log "github.com/sirupsen/logrus" + "github.com/pkg/errors" + "go.uber.org/zap" "github.com/fission/fission/redis/build/gen" ) -func NewClient() redis.Conn { +func NewClient() (redis.Conn, error) { redisIP := os.Getenv("REDIS_SERVICE_HOST") // TODO: Do this here or somewhere earlier? redisPort := os.Getenv("REDIS_SERVICE_PORT") - redisUrl := fmt.Sprintf("%s:%s", redisIP, redisPort) - if len(redisUrl) == 0 { - log.Error("Could not reach Redis in cluster at IP ", redisUrl) - return nil + if len(redisIP) == 0 || len(redisPort) == 0 { + return nil, errors.New("redis host or port not supplied") } - c, err := redis.Dial("tcp", redisUrl) + redisURL := fmt.Sprintf("%s:%s", redisIP, redisPort) + + c, err := redis.Dial("tcp", redisURL) if err != nil { - log.Errorf("Could not connect to Redis: %v\n", err) - return nil + return nil, errors.Wrapf(err, "could not connect to Redis at url %q", redisURL) } - return c + return c, nil } -func Record(triggerName string, recorderName string, reqUID string, request *http.Request, originalUrl url.URL, payload string, response *http.Response, namespace string, timestamp int64) { +func Record(logger *zap.Logger, triggerName string, recorderName string, reqUID string, request *http.Request, originalUrl url.URL, payload string, response *http.Response, namespace string, timestamp int64) { // Case where the function should not have been recorded if len(reqUID) == 0 { return @@ -58,8 +58,9 @@ func Record(triggerName string, recorderName string, reqUID string, request *htt fullPath := originalUrl.String() escPayload := string(json.RawMessage(payload)) - client := NewClient() - if client == nil { + client, err := NewClient() + if err != nil { + logger.Error("could not create redis client", zap.Error(err)) return } @@ -92,7 +93,7 @@ func Record(triggerName string, recorderName string, reqUID string, request *htt PostForm: postForm, } - log.Info("Storing request > ", req) + logger.Info("storing request", zap.Any("request", req)) resp := &redisCache.Response{ Status: response.Status, @@ -107,19 +108,19 @@ func Record(triggerName string, recorderName string, reqUID string, request *htt data, err := proto.Marshal(ureq) if err != nil { - log.Error("Error marshalling request: ", err) + logger.Error("error marshalling request", zap.Error(err)) return } _, err = client.Do("HMSET", reqUID, "ReqResponse", data, "Timestamp", timestamp, "Trigger", triggerName) if err != nil { - log.Error("Error saving request: ", err) + logger.Error("error saving request", zap.Error(err)) return } _, err = client.Do("LPUSH", recorderName, reqUID) if err != nil { - log.Error("Error saving recorder-request pair: ", err) + logger.Error("error saving recorder-request pair", zap.Error(err)) return } } diff --git a/redis/redisApi.go b/redis/redisApi.go index 95539031..7b16cad6 100644 --- a/redis/redisApi.go +++ b/redis/redisApi.go @@ -19,7 +19,6 @@ package redis import ( "bytes" "encoding/json" - "errors" "fmt" "io/ioutil" "net/http" @@ -27,18 +26,20 @@ import ( "strings" "time" + "go.uber.org/zap" + "github.com/golang/protobuf/proto" "github.com/gomodule/redigo/redis" - log "github.com/sirupsen/logrus" + "github.com/pkg/errors" "github.com/fission/fission/crd" "github.com/fission/fission/redis/build/gen" ) -func RecordsListAll() ([]byte, error) { - client := NewClient() - if client == nil { - return nil, errors.New("failed to create redis client") +func RecordsListAll(logger *zap.Logger) ([]byte, error) { + client, err := NewClient() + if err != nil { + return nil, errors.Wrap(err, "failed to create redis client") } iter := 0 @@ -59,12 +60,12 @@ func RecordsListAll() ([]byte, error) { if strings.HasPrefix(key, "REQ") { val, err := redis.Bytes(client.Do("HGET", key, "ReqResponse")) if err != nil { - log.Error("Error retrieving request from Redis: ", err) + logger.Error("error retrieving request from redis", zap.Error(err)) return nil, err } entry, err := deserializeReqResponse(val, key) if err != nil { - log.Error("Error deserializing request: ", err) + logger.Error("error deserializing request from redis", zap.Error(err)) return nil, err } filtered = append(filtered, entry) @@ -84,18 +85,22 @@ func RecordsListAll() ([]byte, error) { // Input: `from` (hours ago, between 0 [today] and 5) and `to` (same units) // Note: Fractional values don't seem to work -- document that for the user -func RecordsFilterByTime(from string, to string) ([]byte, error) { +func RecordsFilterByTime(logger *zap.Logger, from string, to string) ([]byte, error) { rangeStart, rangeEnd, err := obtainInterval(from, to) - log.Debug("Interval inferred: ", rangeStart, rangeEnd) - - if rangeStart >= rangeEnd { - log.Error("Invalid chronology") + if err != nil { return nil, err } + logger.Debug("interval inferred", zap.Int64("range_start", rangeStart), zap.Int64("range_end", rangeEnd)) - client := NewClient() + if rangeStart >= rangeEnd { + e := "invalid chronology - start is greater than or equal to end" + logger.Error(e, zap.Int64("range_start", rangeStart), zap.Int64("range_end", rangeEnd)) + return nil, errors.New(e) + } + + client, err := NewClient() if client == nil { - return nil, errors.New("failed to create redis client") + return nil, errors.Wrap(err, "failed to create redis client") } iter := 0 @@ -114,24 +119,24 @@ func RecordsFilterByTime(from string, to string) ([]byte, error) { if strings.HasPrefix(key, "REQ") { val, err := redis.Strings(client.Do("HMGET", key, "Timestamp")) if err != nil { - log.Error("Error retrieving timestamp from Redis: ", err) + logger.Error("error retrieving timestamp from redis", zap.Error(err)) return nil, err } tsO, err := strconv.Atoi(val[0]) if err != nil { - log.Error("Error converting timestamp to int: ", err) + logger.Error("error converting timestamp to int", zap.Error(err)) return nil, err } ts := int64(tsO) if ts >= rangeStart && ts <= rangeEnd { val2, err := redis.Bytes(client.Do("HGET", key, "ReqResponse")) if err != nil { - log.Error("Error retrieving request from Redis: ", err) + logger.Error("error retrieving request from redis", zap.Error(err)) return nil, err } entry, err := deserializeReqResponse(val2, key) if err != nil { - log.Error("Error deserializing request: ", err) + logger.Error("error deserializing request from redis", zap.Error(err)) return nil, err } filtered = append(filtered, entry) @@ -151,7 +156,7 @@ func RecordsFilterByTime(from string, to string) ([]byte, error) { return resp, nil } -func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) { +func RecordsFilterByTrigger(logger *zap.Logger, queriedTriggerName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) { matchingRecorders := make(map[string]bool) // Implicit triggers: @@ -178,9 +183,9 @@ func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderLi } } - client := NewClient() - if client == nil { - return nil, errors.New("failed to create redis client") + client, err := NewClient() + if err != nil { + return nil, errors.Wrap(err, "failed to create redis client") } var filtered []*redisCache.RecordedEntry @@ -195,19 +200,19 @@ func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderLi for _, reqUID := range val { val, err := redis.Strings(client.Do("HMGET", reqUID, "Trigger")) // 1-to-1 reqUID - trigger? if err != nil { - log.Error("Error retrieving trigger for a request from Redis: ", err) + logger.Error("error retrieving trigger for a request from redis", zap.Error(err)) return nil, err } if val[0] == queriedTriggerName { // TODO: Reconsider multiple commands val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse")) if err != nil { - log.Error("Error retrieving request from Redis: ", err) + logger.Error("error retrieving request from redis", zap.Error(err)) return nil, err } entry, err := deserializeReqResponse(val, reqUID) if err != nil { - log.Error("Error deserializing request: ", err) + logger.Error("error deserializing request from redis", zap.Error(err)) return nil, err } filtered = append(filtered, entry) @@ -222,7 +227,7 @@ func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderLi return resp, nil } -func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) { +func RecordsFilterByFunction(logger *zap.Logger, queriedFunctionName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) { // Implicit functions: // Sometimes functions are not explicitly attached to recorders but we still want to be able to @@ -251,9 +256,9 @@ func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.Recorder } } - client := NewClient() - if client == nil { - return nil, errors.New("failed to create redis client") + client, err := NewClient() + if err != nil { + return nil, errors.Wrap(err, "failed to create redis client") } var filtered []*redisCache.RecordedEntry @@ -273,12 +278,12 @@ func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.Recorder if exists > 0 { val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse")) if err != nil { - log.Error("Error retrieving request from Redis: ", err) + logger.Error("error retrieving request from redis", zap.Error(err)) return nil, err } entry, err := deserializeReqResponse(val, reqUID) if err != nil { - log.Error("Error deserializing request: ", err) + logger.Error("error deserializing request from redis", zap.Error(err)) return nil, err } filtered = append(filtered, entry) @@ -298,10 +303,8 @@ func deserializeReqResponse(value []byte, reqUID string) (*redisCache.RecordedEn data := &redisCache.UniqueRequest{} err := proto.Unmarshal(value, data) if err != nil { - log.Error("Error unmarshalling request: ", err) - return nil, err + return nil, errors.Wrap(err, "error unmarshalling request") } - log.Info("Parsed protobuf bytes: ", data) transformed := &redisCache.RecordedEntry{ ReqUID: reqUID, Req: data.Req, @@ -341,38 +344,38 @@ func includesTrigger(triggers []string, query string) bool { return false } -func ReplayByReqUID(routerUrl string, queriedID string) ([]byte, error) { - client := NewClient() - if client == nil { - return nil, errors.New("failed to create redis client") +func ReplayByReqUID(logger *zap.Logger, routerUrl string, queriedID string) ([]byte, error) { + client, err := NewClient() + if err != nil { + return nil, errors.Wrap(err, "failed to create redis client") } exists, err := redis.Int(client.Do("EXISTS", queriedID)) if exists != 1 || err != nil { - log.Error("couldn't find request to replay") + logger.Error("could not find request to replay in redis", zap.Error(err)) return nil, err } val, err := redis.Bytes(client.Do("HGET", queriedID, "ReqResponse")) if err != nil { - log.Error("couldn't obtain ReqResponse for this ID") + logger.Error("could not obtain ReqResponse for ID from redis", zap.Error(err), zap.String("id", queriedID)) return nil, err } entry, err := deserializeReqResponse(val, queriedID) if err != nil { - log.Error("couldn't deserialize ReqResponse") + logger.Error("error deserializing request from redis", zap.Error(err)) return nil, err } replayed, err := ReplayRequest(routerUrl, entry.Req) if err != nil { - log.Error("couldn't replay request") + logger.Error("error replaying request", zap.Error(err)) return nil, err } resp, err := json.Marshal(replayed) if err != nil { - log.Error("couldn't marshall replayed request response") + logger.Error("error marshalling replayed request response", zap.Error(err)) return nil, err } diff --git a/router/functionHandler.go b/router/functionHandler.go index 36af7e92..10d574da 100644 --- a/router/functionHandler.go +++ b/router/functionHandler.go @@ -33,8 +33,8 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/satori/go.uuid" - log "github.com/sirupsen/logrus" "go.opencensus.io/plugin/ochttp" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" @@ -51,6 +51,7 @@ const ( type ( functionHandler struct { + logger *zap.Logger fmap *functionServiceMap frmap *functionRecorderMap trmap *triggerRecorderMap @@ -86,6 +87,7 @@ type ( // A layer on top of http.DefaultTransport, with retries. RetryingRoundTripper struct { + logger *zap.Logger funcHandler *functionHandler base http.RoundTripper } @@ -146,7 +148,7 @@ func (w *fakeCloseReadCloser) RealClose() error { // if it returned an error. func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) { // Set forwarded host header if not exists - addForwardedHostHeader(req) + roundTripper.addForwardedHostHeader(req) // TODO: Keep? --> Needed for queries encoded in URL before they're stripped by the proxy var originalUrl url.URL @@ -164,7 +166,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt rdr1.Read(p) postedBody = string(p) - log.Info(fmt.Sprintf("%v", postedBody)) + roundTripper.logger.Info("roundtripper posted body", zap.String("body", postedBody)) req.Body = rdr2 } } @@ -287,13 +289,15 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt if roundTripper.funcHandler.httpTrigger != nil { trigger := roundTripper.funcHandler.httpTrigger.Metadata.Name redis.Record( + roundTripper.logger, trigger, roundTripper.funcHandler.recorderName, req.Header.Get("X-Fission-ReqUID"), req, originalUrl, postedBody, resp, fnMeta.Namespace, time.Now().UnixNano(), ) } else { - log.Printf("No http trigger attached for recorder: %v", roundTripper.funcHandler.recorderName) + roundTripper.logger.Error("no http trigger attached for recorder", + zap.String("recorder", roundTripper.funcHandler.recorderName)) } } @@ -303,7 +307,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt // if transport.RoundTrip returns a non-network dial error, then relay it back to user if !fission.IsNetworkDialError(err) { - err = errors.Wrapf(err, "Error sending request to function %v", fnMeta.Name) + err = errors.Wrapf(err, "error sending request to function %v", fnMeta.Name) return resp, err } @@ -313,8 +317,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt executingTimeout = executingTimeout * time.Duration(roundTripper.funcHandler.tsRoundTripperParams.timeoutExponent) retryCounter++ - log.Printf("request to %s errored out. backing off for %v before retrying", - req.URL.Host, executingTimeout) + roundTripper.logger.Info("request errored out - backing off before retrying", + zap.String("url", req.URL.Host), + zap.Duration("backoff_timeout", executingTimeout)) time.Sleep(executingTimeout) @@ -324,9 +329,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt } else { // if transport.RoundTrip returns a network dial error and serviceUrl was from cache, // it means, the entry in router cache is stale, so invalidate it. - log.Printf("request to %s errored out. removing function : %s from router's cache "+ - "and requesting a new service for function", - req.URL.Host, fnMeta.Name) + roundTripper.logger.Error("request errored out - removing function from router's cache and requesting a new service for function", + zap.String("url", req.URL.Host), + zap.String("function_name", fnMeta.Name)) roundTripper.funcHandler.fmap.remove(fnMeta) retryCounter = 0 } @@ -340,7 +345,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt // finally, one more retry with the default timeout resp, err = http.DefaultTransport.RoundTrip(req) if err != nil { - log.Printf("Error getting response from function %v: %v", fnMeta.Name, err) + roundTripper.logger.Error("error getting response from function", + zap.Error(err), + zap.String("function_name", fnMeta.Name)) } return resp, err @@ -385,19 +392,19 @@ func (fh functionHandler) handler(responseWriter http.ResponseWriter, request *h UID := strings.ToLower(uuid.NewV4().String()) reqUID = "REQ" + UID request.Header.Set("X-Fission-ReqUID", reqUID) - log.Print("Record request with ReqUID: ", reqUID) + fh.logger.Info("record request", zap.String("request_id", reqUID)) } if fh.httpTrigger != nil && fh.httpTrigger.Spec.FunctionReference.Type == fission.FunctionReferenceTypeFunctionWeights { // canary deployment. need to determine the function to send request to now fnMetadata := getCanaryBackend(fh.functionMetadataMap, fh.fnWeightDistributionList) if fnMetadata == nil { - log.Printf("Error getting canary backend ") + fh.logger.Error("could not get canary backend", zap.String("request_id", reqUID)) // TODO : write error to responseWrite and return response return } fh.function = fnMetadata - log.Debugf("chosen fnBackend's metadata : %+v", fh.function) + fh.logger.Debug("chosen function backend's metadata", zap.Any("metadata", fh.function)) } // system params @@ -413,6 +420,7 @@ func (fh functionHandler) handler(responseWriter http.ResponseWriter, request *h proxy := &httputil.ReverseProxy{ Director: director, Transport: &RetryingRoundTripper{ + logger: fh.logger.Named("roundtripper"), funcHandler: &fh, base: &ochttp.Transport{ Base: &http.Transport{ @@ -471,7 +479,7 @@ func getCanaryBackend(fnMetadatamap map[string]*metav1.ObjectMeta, fnWtDistribut } // addForwardedHostHeader add "forwarded host" to request header -func addForwardedHostHeader(req *http.Request) { +func (roundTripper RetryingRoundTripper) addForwardedHostHeader(req *http.Request) { // for more detailed information, please visit: // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded @@ -486,7 +494,9 @@ func addForwardedHostHeader(req *http.Request) { reqUrl := fmt.Sprintf("%s://%s", req.Proto, req.Host) u, err := url.Parse(reqUrl) if err != nil { - log.Printf("Error parsing request url (%v): %v", reqUrl, err) + roundTripper.logger.Error("error parsing request url while adding forwarded host headers", + zap.Error(err), + zap.String("url", reqUrl)) return } @@ -531,14 +541,19 @@ func (fh *functionHandler) getServiceEntry(ctx context.Context) (serviceUrl *url var u *url.URL // Get service entry from executor and update cache if its the first goroutine if firstToTheLock { // first to the service url - log.Printf("Calling getServiceForFunction for function: %s", fh.function.Name) + fh.logger.Info("calling getServiceForFunction", + zap.String("function_name", fh.function.Name)) u, err = fh.getServiceEntryFromExecutor(ctx) if err != nil { - log.Printf("Error getting service url from executor: %v", err) + fh.logger.Error("error getting service url from executor", + zap.Error(err), + zap.String("function_name", fh.function.Name)) return nil, err } // add the address in router's cache - log.Printf("Assigning service url: %s for function: %s", u, fh.function.Name) + fh.logger.Info("assigning service url for function", + zap.String("url", u.String()), + zap.String("function_name", fh.function.Name)) fh.fmap.assign(fh.function, u) } else { u, err = fh.getServiceEntryFromCache() @@ -554,9 +569,12 @@ func (fh *functionHandler) getServiceEntry(ctx context.Context) (serviceUrl *url }, ) if err != nil { - err = errors.Wrap(err, fmt.Sprintf("Error updating service address entry for function %v_%v", fh.function.Name, fh.function.Namespace)) - log.Println(err) - return nil, false, err + e := "error updating service address entry for function" + fh.logger.Error(e, + zap.Error(err), + zap.String("function_name", fh.function.Name), + zap.String("function_namespace", fh.function.Namespace)) + return nil, false, errors.Wrapf(err, "%s %s_%s", e, fh.function.Name, fh.function.Namespace) } record, ok := recordObj.(svcEntryRecord) @@ -596,14 +614,20 @@ func (fh *functionHandler) getServiceEntryFromExecutor(ctx context.Context) (*ur service, err := fh.executor.GetServiceForFunction(ctx, fh.function) if err != nil { statusCode, errMsg := fission.GetHTTPError(err) - log.Printf("Error from GetServiceForFunction for function (%v): %v : %v", fh.function, statusCode, errMsg) + fh.logger.Error("error from GetServiceForFunction", + zap.Error(err), + zap.String("error_message", errMsg), + zap.Any("function", fh.function), + zap.Int("status_code", statusCode)) return nil, err } // parse the address into url serviceUrl, err := url.Parse(fmt.Sprintf("http://%v", service)) if err != nil { - log.Printf("Error parsing service url (%v): %v", serviceUrl, err) + fh.logger.Error("error parsing service url", + zap.Error(err), + zap.String("service_url", serviceUrl.String())) return nil, err } diff --git a/router/functionHandler_test.go b/router/functionHandler_test.go index f8ded54b..d1887c0a 100644 --- a/router/functionHandler_test.go +++ b/router/functionHandler_test.go @@ -24,9 +24,11 @@ import ( "testing" "time" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/fission/fission" "github.com/fission/fission/crd" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func createBackendService(testResponseString string) *url.URL { @@ -53,8 +55,10 @@ func TestFunctionProxying(t *testing.T) { log.Printf("Created backend svc at %v", backendURL) fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault} + logger, err := zap.NewDevelopment() + panicIf(err) - fmap := makeFunctionServiceMap(0) + fmap := makeFunctionServiceMap(logger, 0) fmap.assign(fn, backendURL) httpTrigger := &crd.HTTPTrigger{ @@ -70,7 +74,9 @@ func TestFunctionProxying(t *testing.T) { }, } - fh := &functionHandler{fmap: fmap, + fh := &functionHandler{ + logger: logger, + fmap: fmap, function: fn, tsRoundTripperParams: &tsRoundTripperParams{ timeout: 50 * time.Millisecond, diff --git a/router/functionRecorderMap.go b/router/functionRecorderMap.go index ca66edd8..144b2281 100644 --- a/router/functionRecorderMap.go +++ b/router/functionRecorderMap.go @@ -17,9 +17,10 @@ limitations under the License. package router import ( - "log" "time" + "go.uber.org/zap" + "github.com/fission/fission" "github.com/fission/fission/cache" "github.com/fission/fission/crd" @@ -27,14 +28,16 @@ import ( type ( functionRecorderMap struct { - cache *cache.Cache // map[string]*crd.Recorder + logger *zap.Logger + cache *cache.Cache // map[string]*crd.Recorder } ) // Why do we need an expiry? -func makeFunctionRecorderMap(expiry time.Duration) *functionRecorderMap { +func makeFunctionRecorderMap(logger *zap.Logger, expiry time.Duration) *functionRecorderMap { return &functionRecorderMap{ - cache: cache.MakeCache(expiry, 0), + logger: logger.Named("function_recorder_map"), + cache: cache.MakeCache(expiry, 0), } } @@ -53,7 +56,7 @@ func (frmap *functionRecorderMap) assign(function string, recorder *crd.Recorder if e, ok := err.(fission.Error); ok && e.Code == fission.ErrorNameExists { return } - log.Printf("error caching recorder for function name with a different value: %v", err) + frmap.logger.Error("error caching recorder for function name with a different value", zap.Error(err)) } } diff --git a/router/functionServiceMap.go b/router/functionServiceMap.go index 74d47b14..414e6bc9 100644 --- a/router/functionServiceMap.go +++ b/router/functionServiceMap.go @@ -17,10 +17,10 @@ limitations under the License. package router import ( - "log" "net/url" "time" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission/cache" @@ -28,7 +28,8 @@ import ( type ( functionServiceMap struct { - cache *cache.Cache // map[metadataKey]*url.URL + logger *zap.Logger + cache *cache.Cache // map[metadataKey]*url.URL } // metav1.ObjectMeta is not hashable, so we make a hashable copy @@ -40,9 +41,10 @@ type ( } ) -func makeFunctionServiceMap(expiry time.Duration) *functionServiceMap { +func makeFunctionServiceMap(logger *zap.Logger, expiry time.Duration) *functionServiceMap { return &functionServiceMap{ - cache: cache.MakeCache(expiry, 0), + logger: logger.Named("function_service_map"), + cache: cache.MakeCache(expiry, 0), } } @@ -71,7 +73,7 @@ func (fmap *functionServiceMap) assign(f *metav1.ObjectMeta, serviceUrl *url.URL if *serviceUrl == *(old.(*url.URL)) { return } - log.Printf("error caching service url for function with a different value: %v", err) + fmap.logger.Error("error caching service url for function with a different value", zap.Error(err)) // ignore error } } diff --git a/router/functionServiceMap_test.go b/router/functionServiceMap_test.go index db395ff5..d097a295 100644 --- a/router/functionServiceMap_test.go +++ b/router/functionServiceMap_test.go @@ -20,11 +20,16 @@ import ( "net/url" "testing" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestFunctionServiceMap(t *testing.T) { - m := makeFunctionServiceMap(0) + logger, err := zap.NewDevelopment() + panicIf(err) + + m := makeFunctionServiceMap(logger, 0) fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault} u, err := url.Parse("/foo012") if err != nil { diff --git a/router/httpTriggers.go b/router/httpTriggers.go index 12be67ea..14f9c54f 100644 --- a/router/httpTriggers.go +++ b/router/httpTriggers.go @@ -18,11 +18,11 @@ package router import ( "context" - "log" "net/http" "time" "github.com/gorilla/mux" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" @@ -39,6 +39,7 @@ type HTTPTriggerSet struct { *functionServiceMap *mutableRouter + logger *zap.Logger fissionClient *crd.FissionClient kubeClient *kubernetes.Clientset executor *executorClient.Client @@ -57,10 +58,11 @@ type HTTPTriggerSet struct { svcAddrUpdateThrottler *throttler.Throttler } -func makeHTTPTriggerSet(fmap *functionServiceMap, frmap *functionRecorderMap, trmap *triggerRecorderMap, fissionClient *crd.FissionClient, +func makeHTTPTriggerSet(logger *zap.Logger, fmap *functionServiceMap, frmap *functionRecorderMap, trmap *triggerRecorderMap, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient *rest.RESTClient, params *tsRoundTripperParams, isDebugEnv bool, actionThrottler *throttler.Throttler) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) { httpTriggerSet := &HTTPTriggerSet{ + logger: logger.Named("http_trigger_set"), functionServiceMap: fmap, triggers: []crd.HTTPTrigger{}, fissionClient: fissionClient, @@ -83,7 +85,7 @@ func makeHTTPTriggerSet(fmap *functionServiceMap, frmap *functionRecorderMap, tr httpTriggerSet.funcStore = fnStore httpTriggerSet.funcController = fnController } - recorderSet = MakeRecorderSet(httpTriggerSet, crdClient, rStore, frmap, trmap) + recorderSet = MakeRecorderSet(logger, httpTriggerSet, crdClient, rStore, frmap, trmap) httpTriggerSet.recorderSet = recorderSet return httpTriggerSet, tStore, fnStore } @@ -95,7 +97,7 @@ func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter if ts.fissionClient == nil { // Used in tests only. - log.Printf("Skipping continuous trigger updates") + ts.logger.Info("skipping continuous trigger updates") return } go ts.updateRouter() @@ -104,7 +106,7 @@ func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter if ts.recorderSet.recController != nil { go ts.runWatcher(ctx, ts.recorderSet.recController) } else { - log.Fatal("Failed to run recorder Controller") + ts.logger.Fatal("failed to run recorder controller") } } @@ -143,10 +145,11 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router { if rr.resolveResultType != resolveResultSingleFunction && rr.resolveResultType != resolveResultMultipleFunctions { // not implemented yet - log.Panicf("resolve result type not implemented (%v)", rr.resolveResultType) + ts.logger.Panic("resolve result type not implemented", zap.Any("type", rr.resolveResultType)) } fh := &functionHandler{ + logger: ts.logger.Named(trigger.Metadata.Name), fmap: ts.functionServiceMap, frmap: ts.recorderSet.functionRecorderMap, trmap: ts.recorderSet.triggerRecorderMap, @@ -205,6 +208,7 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router { } fh := &functionHandler{ + logger: ts.logger.Named(m.Name), fmap: ts.functionServiceMap, frmap: ts.recorderSet.functionRecorderMap, trmap: ts.recorderSet.triggerRecorderMap, @@ -235,7 +239,7 @@ func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Cont k8sCache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { trigger := obj.(*crd.HTTPTrigger) - go createIngress(trigger, ts.kubeClient) + go createIngress(ts.logger, trigger, ts.kubeClient) ts.syncTriggers() // Check if this trigger's function needs to be recorded fnRef := trigger.Spec.FunctionReference.Name @@ -244,14 +248,17 @@ func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Cont if len(recorder.Spec.Triggers) == 0 { ts.recorderSet.triggerRecorderMap.assign(trigger.Metadata.Name, recorder) } + } else if err != nil { + ts.logger.Error("unable to lookup function in functionRecorderMap", zap.Error(err)) } else { - log.Print("Unable to lookup function in functionRecorderMap") + ts.logger.Error("unable to lookup function in functionRecorderMap") + } }, DeleteFunc: func(obj interface{}) { ts.syncTriggers() trigger := obj.(*crd.HTTPTrigger) - go deleteIngress(trigger, ts.kubeClient) + go deleteIngress(ts.logger, trigger, ts.kubeClient) go ts.recorderSet.DeleteTriggerFromRecorderMap(trigger) }, UpdateFunc: func(oldObj interface{}, newObj interface{}) { @@ -262,7 +269,7 @@ func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Cont return } - go updateIngress(oldTrigger, newTrigger, ts.kubeClient) + go updateIngress(ts.logger, oldTrigger, newTrigger, ts.kubeClient) ts.syncTriggers() }, }) @@ -296,10 +303,10 @@ func (ts *HTTPTriggerSet) initFunctionController() (k8sCache.Store, k8sCache.Con rr.functionMetadataMap[fn.Metadata.Name] != nil && rr.functionMetadataMap[fn.Metadata.Name].ResourceVersion != fn.Metadata.ResourceVersion { // invalidate resolver cache - log.Printf("Invalidating resolver cache") + ts.logger.Info("invalidating resolver cache") err := ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion) if err != nil { - log.Printf("Error deleting functionReferenceResolver cache: %v", err) + ts.logger.Error("error deleting functionReferenceResolver cache", zap.Error(err)) } break diff --git a/router/ingress.go b/router/ingress.go index 0471b245..28ae3477 100644 --- a/router/ingress.go +++ b/router/ingress.go @@ -17,9 +17,9 @@ limitations under the License. package router import ( - "log" "os" + "go.uber.org/zap" "k8s.io/api/extensions/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,16 +38,16 @@ func init() { } } -func createIngress(trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { +func createIngress(logger *zap.Logger, trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { if !trigger.Spec.CreateIngress { - log.Printf("Skipping creation of ingress for trigger: %v", trigger.Metadata.Name) + logger.Info("skipping creation of ingress for trigger", zap.String("trigger", trigger.Metadata.Name)) return } _, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{}) if err == nil { - log.Printf("Ingress for trigger exists already %v", trigger.Metadata.Name) + logger.Info("ingress for trigger exists already", zap.String("trigger", trigger.Metadata.Name)) return } @@ -87,10 +87,10 @@ func createIngress(trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { _, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Create(ing) if err != nil { - log.Printf("Failed to create ingress: %v", err) + logger.Error("failed to create ingress", zap.Error(err)) return } - log.Printf("Created ingress successfully for trigger %v", trigger.Metadata.Name) + logger.Info("created ingress successfully for trigger", zap.String("trigger", trigger.Metadata.Name)) } func getDeployLabels(trigger *crd.HTTPTrigger) map[string]string { @@ -101,41 +101,46 @@ func getDeployLabels(trigger *crd.HTTPTrigger) map[string]string { } } -func deleteIngress(trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { +func deleteIngress(logger *zap.Logger, trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { if !trigger.Spec.CreateIngress { return } ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{}) if err != nil { - log.Printf("Failed to get ingress when deleting trigger: %v, %v", err, trigger) + logger.Error("failed to get ingress when deleting trigger", zap.Error(err), zap.String("trigger", trigger.Metadata.Name)) } err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Delete(ingress.Name, &v1.DeleteOptions{}) if err != nil { - log.Printf("Failed to delete ingress %v error: %v", ingress, err) + logger.Error("failed to delete ingress for trigger", + zap.Error(err), + zap.Any("ingress", ingress), + zap.String("trigger", trigger.Metadata.Name)) } } -func updateIngress(oldT *crd.HTTPTrigger, newT *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { +func updateIngress(logger *zap.Logger, oldT *crd.HTTPTrigger, newT *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { if oldT.Spec.CreateIngress == false && newT.Spec.CreateIngress == true { - createIngress(newT, kubeClient) + createIngress(logger, newT, kubeClient) return } if newT.Spec.CreateIngress == false && oldT.Spec.CreateIngress == true { - deleteIngress(oldT, kubeClient) + deleteIngress(logger, oldT, kubeClient) return } if newT.Spec.Host != oldT.Spec.Host || newT.Spec.RelativeURL != oldT.Spec.RelativeURL { - log.Printf("Updating ingress for trigger %v", oldT.Metadata.Name) + logger.Info("updating ingress for trigger", zap.String("trigger", oldT.Metadata.Name)) ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(oldT.Metadata.Name, v1.GetOptions{}) if err != nil { - log.Printf("Failed to get ingress when updating trigger: %v", err) + logger.Error("failed to get ingress when updating trigger", + zap.Error(err), + zap.String("trigger", oldT.Metadata.Name)) } if newT.Spec.Host != oldT.Spec.Host { @@ -148,7 +153,7 @@ func updateIngress(oldT *crd.HTTPTrigger, newT *crd.HTTPTrigger, kubeClient *kub _, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Update(ingress) if err != nil { - log.Printf("Failed to update ingress for trigger: %v", err) + logger.Error("failed to update ingress for trigger", zap.String("trigger", oldT.Metadata.Name)) } } diff --git a/router/mutablemux.go b/router/mutablemux.go index 525ebd50..ee35cc7b 100644 --- a/router/mutablemux.go +++ b/router/mutablemux.go @@ -17,11 +17,11 @@ limitations under the License. package router import ( - "log" "net/http" "sync/atomic" "github.com/gorilla/mux" + "go.uber.org/zap" ) // @@ -30,11 +30,14 @@ import ( // type mutableRouter struct { + logger *zap.Logger router atomic.Value // mux.Router } -func NewMutableRouter(handler *mux.Router) *mutableRouter { - mr := mutableRouter{} +func NewMutableRouter(logger *zap.Logger, handler *mux.Router) *mutableRouter { + mr := mutableRouter{ + logger: logger.Named("mutable_router"), + } mr.router.Store(handler) return &mr } @@ -44,7 +47,7 @@ func (mr *mutableRouter) ServeHTTP(responseWriter http.ResponseWriter, request * routerValue := mr.router.Load() router, ok := routerValue.(*mux.Router) if !ok { - log.Panic("Invalid router type") + mr.logger.Panic("invalid router type") } router.ServeHTTP(responseWriter, request) } diff --git a/router/mutablemux_test.go b/router/mutablemux_test.go index a8e7df31..97ff9dd8 100644 --- a/router/mutablemux_test.go +++ b/router/mutablemux_test.go @@ -17,11 +17,13 @@ limitations under the License. package router import ( - "github.com/gorilla/mux" "log" "net/http" "testing" "time" + + "github.com/gorilla/mux" + "go.uber.org/zap" ) func OldHandler(responseWriter http.ResponseWriter, request *http.Request) { @@ -62,7 +64,10 @@ func TestMutableMux(t *testing.T) { log.Print("Create mutable router") muxRouter := mux.NewRouter() muxRouter.HandleFunc("/", OldHandler) - mr := NewMutableRouter(muxRouter) + logger, err := zap.NewDevelopment() + panicIf(err) + + mr := NewMutableRouter(logger, muxRouter) // start http server log.Print("Start http server") diff --git a/router/recorderController.go b/router/recorderController.go index 90a6b63e..39be463d 100644 --- a/router/recorderController.go +++ b/router/recorderController.go @@ -1,13 +1,16 @@ package router import ( - "github.com/fission/fission/crd" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" "k8s.io/client-go/rest" k8sCache "k8s.io/client-go/tools/cache" + + "github.com/fission/fission/crd" ) type RecorderSet struct { + logger *zap.Logger + httpTriggerSet *HTTPTriggerSet crdClient *rest.RESTClient @@ -19,8 +22,9 @@ type RecorderSet struct { triggerRecorderMap *triggerRecorderMap } -func MakeRecorderSet(httpTriggerSet *HTTPTriggerSet, crdClient *rest.RESTClient, rStore k8sCache.Store, frmap *functionRecorderMap, trmap *triggerRecorderMap) *RecorderSet { +func MakeRecorderSet(logger *zap.Logger, httpTriggerSet *HTTPTriggerSet, crdClient *rest.RESTClient, rStore k8sCache.Store, frmap *functionRecorderMap, trmap *triggerRecorderMap) *RecorderSet { recorderSet := &RecorderSet{ + logger: logger.Named("recorder_set"), httpTriggerSet: httpTriggerSet, crdClient: crdClient, recStore: rStore, @@ -63,12 +67,17 @@ func (rs *RecorderSet) disableRecorder(r *crd.Recorder) { function := r.Spec.Function triggers := r.Spec.Triggers - log.Info("Disabling recorder ", r.Metadata.Name) + rs.logger.Info("disabling recorder", + zap.String("recorder", r.Metadata.Name), + zap.String("function", function)) // Account for function err := rs.functionRecorderMap.remove(function) if err != nil { - log.Error("Error disabling recorder (failed to remove function from functionRecorderMap): ", err) + rs.logger.Error("error disabling recorder (failed to remove function from functionRecorderMap)", + zap.Error(err), + zap.String("recorder", r.Metadata.Name), + zap.String("function", function)) } // Account for explicitly added triggers @@ -76,7 +85,11 @@ func (rs *RecorderSet) disableRecorder(r *crd.Recorder) { for _, trigger := range triggers { err := rs.triggerRecorderMap.remove(trigger) if err != nil { - log.Error("Error disabling recorder (failed to remove triggers from triggerRecorderMap): ", err) + rs.logger.Error("error disabling recorder (failed to remove triggers from triggerRecorderMap)", + zap.Error(err), + zap.String("recorder", r.Metadata.Name), + zap.String("function", function), + zap.String("trigger", trigger)) } } } else { @@ -86,7 +99,11 @@ func (rs *RecorderSet) disableRecorder(r *crd.Recorder) { if trigger.Spec.FunctionReference.Name == function { err := rs.triggerRecorderMap.remove(trigger.Metadata.Name) if err != nil { - log.Error("Failed to remove trigger from triggerRecorderMap: ", err) + rs.logger.Error("failed to remove trigger from triggerRecorderMap", + zap.Error(err), + zap.String("recorder", r.Metadata.Name), + zap.String("function", function), + zap.String("trigger", trigger.Metadata.Name)) } } } @@ -106,13 +123,13 @@ func (rs *RecorderSet) updateRecorder(old *crd.Recorder, newer *crd.Recorder) { func (rs *RecorderSet) DeleteTriggerFromRecorderMap(trigger *crd.HTTPTrigger) { err := rs.triggerRecorderMap.remove(trigger.Metadata.Name) if err != nil { - log.Error("Failed to remove trigger from triggerRecorderMap: ", err) + rs.logger.Error("failed to remove trigger from triggerRecorderMap", zap.Error(err)) } } func (rs *RecorderSet) DeleteFunctionFromRecorderMap(function *crd.Function) { err := rs.functionRecorderMap.remove(function.Metadata.Name) if err != nil { - log.Error("Failed to remove function from functionRecorderMap: ", err) + rs.logger.Error("failed to remove function from functionRecorderMap", zap.Error(err)) } } diff --git a/router/router.go b/router/router.go index c75359d3..52d65d90 100644 --- a/router/router.go +++ b/router/router.go @@ -42,7 +42,6 @@ package router import ( "context" "fmt" - "log" "net/http" "os" "strconv" @@ -52,6 +51,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "go.opencensus.io/plugin/ochttp" "go.opencensus.io/trace" + "go.uber.org/zap" "github.com/fission/fission" "github.com/fission/fission/crd" @@ -63,16 +63,16 @@ import ( // request url ---[trigger]---> Function(name, deployment) ----[deployment]----> Function(name, uid) ----[pool mgr]---> k8s service url -func router(ctx context.Context, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) *mutableRouter { +func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) *mutableRouter { muxRouter := mux.NewRouter() - mr := NewMutableRouter(muxRouter) - muxRouter.Use(fission.LoggingMiddleware) + mr := NewMutableRouter(logger, muxRouter) + muxRouter.Use(fission.LoggingMiddleware(logger)) httpTriggerSet.subscribeRouter(ctx, mr, resolver) return mr } -func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) { - mr := router(ctx, httpTriggerSet, resolver) +func serve(ctx context.Context, logger *zap.Logger, port int, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) { + mr := router(ctx, logger, httpTriggerSet, resolver) url := fmt.Sprintf(":%v", port) http.ListenAndServe(url, &ochttp.Handler{ Handler: mr, @@ -82,77 +82,102 @@ func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolv }) } -func serveMetric() { +func serveMetric(logger *zap.Logger) { // Expose the registered metrics via HTTP. http.Handle("/metrics", promhttp.Handler()) - log.Fatal(http.ListenAndServe(metricAddr, nil)) + err := http.ListenAndServe(metricAddr, nil) + + logger.Fatal("done listening on metrics endpoint", zap.Error(err)) } -func Start(port int, executorUrl string) { +func Start(logger *zap.Logger, port int, executorUrl string) { // setup a signal handler for SIGTERM fission.SetupStackTraceHandler() - fmap := makeFunctionServiceMap(time.Minute) + fmap := makeFunctionServiceMap(logger, time.Minute) - frmap := makeFunctionRecorderMap(time.Minute) + frmap := makeFunctionRecorderMap(logger, time.Minute) - trmap := makeTriggerRecorderMap(time.Minute) + trmap := makeTriggerRecorderMap(logger, time.Minute) fissionClient, kubeClient, _, err := crd.MakeFissionClient() if err != nil { - log.Fatalf("Error connecting to kubernetes API: %v", err) + logger.Fatal("error connecting to kubernetes API", zap.Error(err)) } err = fissionClient.WaitForCRDs() if err != nil { - log.Fatalf("Error waiting for CRDs: %v", err) + logger.Fatal("error waiting for CRDs", zap.Error(err)) } restClient := fissionClient.GetCrdClient() - executor := executorClient.MakeClient(executorUrl) + executor := executorClient.MakeClient(logger, executorUrl) - timeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_TIMEOUT")) + timeoutStr := os.Getenv("ROUTER_ROUND_TRIP_TIMEOUT") + timeout, err := time.ParseDuration(timeoutStr) if err != nil { - log.Fatalf("Failed to parse timeout: %v", err) + logger.Fatal("failed to parse timeout duration from 'ROUTER_ROUND_TRIP_TIMEOUT'", + zap.Error(err), + zap.String("value", timeoutStr)) } - timeoutExponent, err := strconv.Atoi(os.Getenv("ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT")) + timeoutExponentStr := os.Getenv("ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT") + timeoutExponent, err := strconv.Atoi(timeoutExponentStr) if err != nil { - log.Fatalf("Failed to parse timeout exponent: %v", err) + logger.Fatal("failed to parse timeout exponent from 'ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT'", + zap.Error(err), + zap.String("value", timeoutExponentStr)) } - keepAlive, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME")) + keepAliveStr := os.Getenv("ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME") + keepAlive, err := time.ParseDuration(keepAliveStr) if err != nil { - log.Fatalf("Failed to parse keep alive time: %v", err) + logger.Fatal("failed to parse keep alive duration from 'ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME'", + zap.Error(err), + zap.String("value", keepAliveStr)) } - maxRetries, err := strconv.Atoi(os.Getenv("ROUTER_ROUND_TRIP_MAX_RETRIES")) + maxRetriesStr := os.Getenv("ROUTER_ROUND_TRIP_MAX_RETRIES") + maxRetries, err := strconv.Atoi(maxRetriesStr) if err != nil { - log.Fatalf("Failed to parse max retry times: %v", err) + logger.Fatal("failed to parse max retries from 'ROUTER_ROUND_TRIP_MAX_RETRIES'", + zap.Error(err), + zap.String("value", maxRetriesStr)) } - isDebugEnv, err := strconv.ParseBool(os.Getenv("DEBUG_ENV")) + isDebugEnvStr := os.Getenv("DEBUG_ENV") + isDebugEnv, err := strconv.ParseBool(isDebugEnvStr) if err != nil { - log.Fatalf("Failed to parse DEBUG_ENV: %v", err) + logger.Fatal("failed to parse debug env from 'DEBUG_ENV'", + zap.Error(err), + zap.String("value", isDebugEnvStr)) } // svcAddrRetryCount is the max times for RetryingRoundTripper to retry with a specific service address - svcAddrRetryCount, err := strconv.Atoi(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES")) + svcAddrRetryCountStr := os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES") + svcAddrRetryCount, err := strconv.Atoi(svcAddrRetryCountStr) if err != nil { svcAddrRetryCount = 5 - log.Printf("Failed to parse svc address retry conunt, set it to default value(5): %v", err) + logger.Info("failed to parse service address retry count from 'ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES' - set to the default value", + zap.Error(err), + zap.String("value", svcAddrRetryCountStr), + zap.Int("default", svcAddrRetryCount)) } // svcAddrUpdateTimeout is the timeout setting for a goroutine to wait for the update of a service entry. // If the update process cannot be done within the timeout window, consider it failed. + svcAddrUpdateTimeoutStr := os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT") svcAddrUpdateTimeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT")) if err != nil { svcAddrUpdateTimeout = 30 * time.Second - log.Printf("Failed to parse svc address update timeout, set it to default value(30): %v", err) + logger.Info("failed to parse service address update timeout duration from 'ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT' - set to the default value", + zap.Error(err), + zap.String("value", svcAddrUpdateTimeoutStr), + zap.Duration("default", svcAddrUpdateTimeout)) } - triggers, _, fnStore := makeHTTPTriggerSet(fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{ + triggers, _, fnStore := makeHTTPTriggerSet(logger.Named("triggerset"), fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{ timeout: timeout, timeoutExponent: timeoutExponent, keepAlive: keepAlive, @@ -162,10 +187,10 @@ func Start(port int, executorUrl string) { resolver := makeFunctionReferenceResolver(fnStore) - go serveMetric() + go serveMetric(logger) - log.Printf("Starting router at port %v\n", port) + logger.Info("starting router", zap.Int("port", port)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - serve(ctx, port, triggers, resolver) + serve(ctx, logger, port, triggers, resolver) } diff --git a/router/router_test.go b/router/router_test.go index 8e34839d..d4e60adb 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" @@ -43,16 +44,19 @@ func TestRouter(t *testing.T) { testResponseString := "hi" testServiceUrl := createBackendService(testResponseString) + logger, err := zap.NewDevelopment() + panicIf(err) + // set up the cache with this fake service - fmap := makeFunctionServiceMap(0) + fmap := makeFunctionServiceMap(logger, 0) fmap.assign(fn, testServiceUrl) - frmap := makeFunctionRecorderMap(time.Minute) + frmap := makeFunctionRecorderMap(logger, time.Minute) - trmap := makeTriggerRecorderMap(time.Minute) + trmap := makeTriggerRecorderMap(logger, time.Minute) // HTTP trigger set with a trigger for this function - triggers, _, _ := makeHTTPTriggerSet(fmap, frmap, trmap, nil, nil, nil, nil, + triggers, _, _ := makeHTTPTriggerSet(logger, fmap, frmap, trmap, nil, nil, nil, nil, &tsRoundTripperParams{ timeout: 50 * time.Millisecond, timeoutExponent: 2, @@ -95,7 +99,7 @@ func TestRouter(t *testing.T) { port := 4242 ctx, cancel := context.WithCancel(context.Background()) defer cancel() - go serve(ctx, port, triggers, frr) + go serve(ctx, logger, port, triggers, frr) time.Sleep(100 * time.Millisecond) // hit the router diff --git a/router/triggerRecorderMap.go b/router/triggerRecorderMap.go index 59e164d2..48dea4ae 100644 --- a/router/triggerRecorderMap.go +++ b/router/triggerRecorderMap.go @@ -17,9 +17,10 @@ limitations under the License. package router import ( - "log" "time" + "go.uber.org/zap" + "github.com/fission/fission" "github.com/fission/fission/cache" "github.com/fission/fission/crd" @@ -27,13 +28,15 @@ import ( type ( triggerRecorderMap struct { - cache *cache.Cache // map[string]*crd.Recorder + logger *zap.Logger + cache *cache.Cache // map[string]*crd.Recorder } ) -func makeTriggerRecorderMap(expiry time.Duration) *triggerRecorderMap { +func makeTriggerRecorderMap(logger *zap.Logger, expiry time.Duration) *triggerRecorderMap { return &triggerRecorderMap{ - cache: cache.MakeCache(expiry, 0), + logger: logger.Named("trigger_recorder_map"), + cache: cache.MakeCache(expiry, 0), } } @@ -52,7 +55,7 @@ func (trmap *triggerRecorderMap) assign(trigger string, recorder *crd.Recorder) if e, ok := err.(fission.Error); ok && e.Code == fission.ErrorNameExists { return } - log.Printf("error caching recorder for function name with a different value: %v", err) + trmap.logger.Error("error caching recorder for function name with a different value", zap.Error(err)) } } diff --git a/router/util_test.go b/router/util_test.go index 86322b09..f3e3ce0f 100644 --- a/router/util_test.go +++ b/router/util_test.go @@ -28,3 +28,9 @@ func testRequest(targetUrl string, expectedResponse string) { log.Panic("Unexpected response") } } + +func panicIf(err error) { + if err != nil { + log.Panicf("Error: %v", err) + } +} diff --git a/storagesvc/archivePruner.go b/storagesvc/archivePruner.go index f0d8bf4b..80770448 100644 --- a/storagesvc/archivePruner.go +++ b/storagesvc/archivePruner.go @@ -19,13 +19,14 @@ package storagesvc import ( "time" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission/crd" ) type ArchivePruner struct { + logger *zap.Logger crdClient *crd.FissionClient archiveChan chan (string) stowClient *StowClient @@ -34,13 +35,14 @@ type ArchivePruner struct { const defaultPruneInterval int = 60 // in minutes -func MakeArchivePruner(stowClient *StowClient, pruneInterval time.Duration) (*ArchivePruner, error) { +func MakeArchivePruner(logger *zap.Logger, stowClient *StowClient, pruneInterval time.Duration) (*ArchivePruner, error) { crdClient, _, _, err := crd.MakeFissionClient() if err != nil { return nil, err } return &ArchivePruner{ + logger: logger.Named("archive_pruner"), crdClient: crdClient, archiveChan: make(chan string), stowClient: stowClient, @@ -50,15 +52,18 @@ func MakeArchivePruner(stowClient *StowClient, pruneInterval time.Duration) (*Ar // pruneArchives listens to archiveChannel for archive ids that need to be deleted func (pruner *ArchivePruner) pruneArchives() { - log.Info("listening to archiveChannel to prune archives") + pruner.logger.Info("listening to archiveChannel to prune archives") for { select { case archiveID := <-pruner.archiveChan: - log.WithField("archive ID", archiveID).Info("sending delete request") + pruner.logger.Info("sending delete request for archive", + zap.String("archive_id", archiveID)) if err := pruner.stowClient.removeFileByID(archiveID); err != nil { // logging the error and continuing with other deletions. // hopefully this archive will be deleted in the next iteration. - log.WithField("archiveID", archiveID).WithError(err).Error("Ignoring error while deleting archive") + pruner.logger.Error("ignoring error while deleting archive", + zap.Error(err), + zap.String("archive_id", archiveID)) } } } @@ -72,16 +77,16 @@ func (pruner *ArchivePruner) insertArchive(archiveID string) { // A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes // and not the archives that are referenced by them, leaving the archives as orphans. // getOrphanArchives reaps the orphaned archives. -func (pruner *ArchivePruner) getOrphanArchives() error { - log.Info("getting orphan archives") +func (pruner *ArchivePruner) getOrphanArchives() { + pruner.logger.Info("getting orphan archives") archivesRefByPkgs := make([]string, 0) var archiveID string // get all pkgs from kubernetes pkgList, err := pruner.crdClient.Packages(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { - log.WithError(err).Error("Error getting package list from kubernetes") - return err + pruner.logger.Error("error getting package list from kubernetes", zap.Error(err)) + return } // extract archives referenced by these pkgs @@ -89,44 +94,48 @@ func (pruner *ArchivePruner) getOrphanArchives() error { if pkg.Spec.Deployment.URL != "" { archiveID, err = getQueryParamValue(pkg.Spec.Deployment.URL, "id") if err != nil { - log.WithError(err).Error("Error extracting value of archiveID from url") - return err + pruner.logger.Error("error extracting value of archiveID from deployment url", + zap.Error(err), + zap.String("url", pkg.Spec.Deployment.URL)) + return } archivesRefByPkgs = append(archivesRefByPkgs, archiveID) } if pkg.Spec.Source.URL != "" { archiveID, err = getQueryParamValue(pkg.Spec.Source.URL, "id") if err != nil { - log.WithError(err).Error("Error extracting value of archiveID from url") - return err + pruner.logger.Error("error extracting value of archiveID from source url", + zap.Error(err), + zap.String("url", pkg.Spec.Source.URL)) + return } archivesRefByPkgs = append(archivesRefByPkgs, archiveID) } } - log.WithField("list", "archives referenced by packages").Debugf("%s", archivesRefByPkgs) + pruner.logger.Debug("archives referenced by packagese", zap.Strings("archives", archivesRefByPkgs)) // get all archives on storage // out of them, there may be some just created but not referenced by packages yet. // need to filter them out. - archivesInStorage, err := pruner.stowClient.getItemIDsWithFilter(filterItemCreatedAMinuteAgo, time.Now()) + archivesInStorage, err := pruner.stowClient.getItemIDsWithFilter(pruner.stowClient.filterItemCreatedAMinuteAgo, time.Now()) if err != nil { - log.WithError(err).Error("Error getting items from storage") - return err + pruner.logger.Error("error getting items from storage", zap.Error(err)) + return } - log.WithField("list", "archives in storage").Debugf("%s", archivesInStorage) + pruner.logger.Debug("archives in storage", zap.Strings("archives", archivesInStorage)) // difference of the two lists gives us the list of orphan archives. This is just a brute force approach. // need to do something more optimal at scale. orphanedArchives := getDifferenceOfLists(archivesInStorage, archivesRefByPkgs) - log.WithField("list", "orphan archives").Debugf("%s", orphanedArchives) + pruner.logger.Debug("orphan archives", zap.Strings("archives", orphanedArchives)) // send each orphan archive away for deletion for _, archiveID = range orphanedArchives { pruner.insertArchive(archiveID) } - return nil + return } // Start starts a go routine that listens to a channel for archive IDs that need to deleted. diff --git a/storagesvc/client/storagesvc_test.go b/storagesvc/client/storagesvc_test.go index 9649c436..e7337323 100644 --- a/storagesvc/client/storagesvc_test.go +++ b/storagesvc/client/storagesvc_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/dchest/uniuri" + "go.uber.org/zap" "github.com/fission/fission/storagesvc" ) @@ -52,9 +53,12 @@ func TestStorageService(t *testing.T) { port := 8080 enableArchivePruner := false + logger, err := zap.NewDevelopment() + panicIf(err) + log.Println("starting storage svc") _ = storagesvc.RunStorageService( - storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner) + logger, storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner) time.Sleep(time.Second) client := MakeClient(fmt.Sprintf("http://localhost:%v/", port)) diff --git a/storagesvc/storagesvc.go b/storagesvc/storagesvc.go index 1c3067e1..74be79c1 100644 --- a/storagesvc/storagesvc.go +++ b/storagesvc/storagesvc.go @@ -25,15 +25,17 @@ import ( "strconv" "time" - "github.com/fission/fission" "github.com/gorilla/mux" _ "github.com/graymeta/stow/local" - log "github.com/sirupsen/logrus" "go.opencensus.io/plugin/ochttp" + "go.uber.org/zap" + + "github.com/fission/fission" ) type ( StorageService struct { + logger *zap.Logger storageClient *StowClient port int } @@ -61,27 +63,34 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request) fileSizeS, ok := r.Header["X-File-Size"] if !ok { - log.Error("Missing X-File-Size") + ss.logger.Error("upload is missing the 'X-File-Size' header", + zap.String("filename", handler.Filename)) http.Error(w, "missing X-File-Size header", http.StatusBadRequest) return } fileSize, err := strconv.Atoi(fileSizeS[0]) if err != nil { - log.WithError(err).Errorf("Error parsing x-file-size: '%v'", fileSizeS) + ss.logger.Error("error parsing 'X-File-Size' header", + zap.Error(err), + zap.Strings("header", fileSizeS), + zap.String("filename", handler.Filename)) http.Error(w, "missing or bad X-File-Size header", http.StatusBadRequest) return } // TODO: allow headers to add more metadata (e.g. environment // and function metadata) - log.Infof("Handling upload for %v", handler.Filename) + ss.logger.Info("handling upload", + zap.String("filename", handler.Filename)) //fileMetadata := make(map[string]interface{}) //fileMetadata["filename"] = handler.Filename id, err := ss.storageClient.putFile(file, int64(fileSize)) if err != nil { - log.WithError(err).Error("Error saving uploaded file") + ss.logger.Error("error saving uploaded file", + zap.Error(err), + zap.String("filename", handler.Filename)) http.Error(w, "Error saving uploaded file", http.StatusInternalServerError) return } @@ -92,6 +101,9 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request) } resp, err := json.Marshal(ur) if err != nil { + ss.logger.Error("error marshaling uploaded file response", + zap.Error(err), + zap.String("filename", handler.Filename)) http.Error(w, "Error marshaling response", http.StatusInternalServerError) return } @@ -136,7 +148,7 @@ func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request // stream it to response err = ss.storageClient.copyFileToStream(fileId, w) if err != nil { - log.WithError(err).Errorf("Error getting item id '%v'", fileId) + ss.logger.Error("error getting file from storage client", zap.Error(err), zap.String("file_id", fileId)) if err == ErrNotFound { http.Error(w, "Error retrieving item: not found", http.StatusNotFound) } else if err == ErrRetrievingItem { @@ -154,8 +166,9 @@ func (ss *StorageService) healthHandler(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusOK) } -func MakeStorageService(storageClient *StowClient, port int) *StorageService { +func MakeStorageService(logger *zap.Logger, storageClient *StowClient, port int) *StorageService { return &StorageService{ + logger: logger.Named("storage_service"), storageClient: storageClient, port: port, } @@ -170,30 +183,27 @@ func (ss *StorageService) Start(port int) { address := fmt.Sprintf(":%v", port) - r.Use(fission.LoggingMiddleware) + r.Use(fission.LoggingMiddleware(ss.logger)) err := http.ListenAndServe(address, &ochttp.Handler{ Handler: r, // Propagation: &b3.HTTPFormat{}, }) - log.Fatal(err) + ss.logger.Fatal("done listening", zap.Error(err)) } -func RunStorageService(storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService { +func RunStorageService(logger *zap.Logger, storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService { // setup a signal handler for SIGTERM fission.SetupStackTraceHandler() - // initialize logger - log.SetLevel(log.InfoLevel) - // create a storage client - storageClient, err := MakeStowClient(storageType, storagePath, containerName) + storageClient, err := MakeStowClient(logger, storageType, storagePath, containerName) if err != nil { - log.Fatalf("Error creating stowClient: %v", err) + logger.Fatal("error creating stowClient", zap.Error(err)) } // create http handlers - storageService := MakeStorageService(storageClient, port) + storageService := MakeStorageService(logger, storageClient, port) go storageService.Start(port) // enablePruner prevents storagesvc unit test from needing to talk to kubernetes @@ -203,13 +213,13 @@ func RunStorageService(storageType StorageType, storagePath string, containerNam if err != nil { pruneInterval = defaultPruneInterval } - pruner, err := MakeArchivePruner(storageClient, time.Duration(pruneInterval)) + pruner, err := MakeArchivePruner(logger, storageClient, time.Duration(pruneInterval)) if err != nil { - log.Fatalf("Error creating archivePruner: %v", err) + logger.Fatal("error creating archivePruner", zap.Error(err)) } go pruner.Start() } - log.Info("Storage service started") + logger.Info("storage service started") return storageService } diff --git a/storagesvc/stowClient.go b/storagesvc/stowClient.go index e2a87ef1..2110be7e 100644 --- a/storagesvc/stowClient.go +++ b/storagesvc/stowClient.go @@ -17,7 +17,6 @@ limitations under the License. package storagesvc import ( - "errors" "io" "mime/multipart" "os" @@ -25,8 +24,9 @@ import ( "github.com/graymeta/stow" _ "github.com/graymeta/stow/local" + "github.com/pkg/errors" "github.com/satori/go.uuid" - log "github.com/sirupsen/logrus" + "go.uber.org/zap" ) type ( @@ -40,6 +40,7 @@ type ( } StowClient struct { + logger *zap.Logger config *storageConfig location stow.Location container stow.Container @@ -59,7 +60,7 @@ var ( ErrWritingFileIntoResponse = errors.New("unable to copy item into http response") ) -func MakeStowClient(storageType StorageType, storagePath string, containerName string) (*StowClient, error) { +func MakeStowClient(logger *zap.Logger, storageType StorageType, storagePath string, containerName string) (*StowClient, error) { if storageType != StorageTypeLocal { return nil, errors.New("Storage types other than 'local' are not implemented") } @@ -71,13 +72,13 @@ func MakeStowClient(storageType StorageType, storagePath string, containerName s } stowClient := &StowClient{ + logger: logger.Named("stow_client"), config: config, } cfg := stow.ConfigMap{"path": config.localPath} loc, err := stow.Dial("local", cfg) if err != nil { - log.WithError(err).Error("Error initializing storage") return nil, err } stowClient.location = loc @@ -99,7 +100,6 @@ func MakeStowClient(storageType StorageType, storagePath string, containerName s } } if err != nil { - log.WithError(err).Error("Error initializing storage") return nil, err } stowClient.container = con @@ -116,11 +116,13 @@ func (client *StowClient) putFile(file multipart.File, fileSize int64) (string, // save the file to the storage backend item, err := client.container.Put(uploadName, file, int64(fileSize), nil) if err != nil { - log.WithError(err).Errorf("Error writing file: %s on storage", uploadName) + client.logger.Error("error writing file on storage", + zap.Error(err), + zap.String("file", uploadName)) return "", ErrWritingFile } - log.Debugf("Successfully wrote file:%s on storage", uploadName) + client.logger.Debug("successfully wrote file on storage", zap.String("file", uploadName)) return item.ID(), nil } @@ -146,7 +148,7 @@ func (client *StowClient) copyFileToStream(fileId string, w io.Writer) error { return ErrWritingFileIntoResponse } - log.Debugf("successfully wrote file: %s into httpresponse", fileId) + client.logger.Debug("successfully wrote file into httpresponse", zap.String("file", fileId)) return nil } @@ -169,7 +171,7 @@ func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncPara for { items, cursor, err = client.container.Items(stow.NoPrefix, cursor, PaginationSize) if err != nil { - log.WithError(err).Error("Error getting items from container") + errors.Wrap(err, "error getting items from container") return nil, err } @@ -191,10 +193,13 @@ func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncPara // filterItemCreatedAMinuteAgo is one type of filter function that filters out items created less than a minute ago. // More filter functions can be written if needed, as long as they are of type filter -func filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool { +func (client StowClient) filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool { itemLastModTime, _ := item.LastMod() if currentTime.(time.Time).Sub(itemLastModTime) < 1*time.Minute { - log.Debugf("item: %s created less than a minute ago: %v", item.ID(), itemLastModTime) + + client.logger.Debug("item created less than a minute ago", + zap.String("item", item.ID()), + zap.Time("last_modified_time", itemLastModTime)) return true } return false diff --git a/storagesvc/util.go b/storagesvc/util.go index a0fa2f5d..4046a65f 100644 --- a/storagesvc/util.go +++ b/storagesvc/util.go @@ -17,15 +17,15 @@ limitations under the License. package storagesvc import ( - "log" "net/url" + + "github.com/pkg/errors" ) func getQueryParamValue(urlString string, queryParam string) (string, error) { url, err := url.Parse(urlString) if err != nil { - log.Printf("Error parsing URL string: %s into URL", urlString) - return "", err + return "", errors.Wrapf(err, "error parsing URL string %q into URL", urlString) } return url.Query().Get(queryParam), nil } diff --git a/test/test_utils.sh b/test/test_utils.sh index f35e610f..fd8749ed 100755 --- a/test/test_utils.sh +++ b/test/test_utils.sh @@ -193,7 +193,7 @@ helm_install_fission() { ns=f-$id fns=f-func-$id - helmVars=repository=$repo,image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImage=$fluentdImage,logger.fluentdImageTag=$fluentdImageTag,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType,serviceType=$serviceType,preUpgradeChecksImage=$preUpgradeCheckImage,prometheus.server.persistentVolume.enabled=false,prometheus.alertmanager.enabled=false,prometheus.kubeStateMetrics.enabled=false,prometheus.nodeExporter.enabled=false + helmVars=repository=$repo,image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImageRepository=$repo,logger.fluentdImage=$fluentdImage,logger.fluentdImageTag=$fluentdImageTag,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType,serviceType=$serviceType,preUpgradeChecksImage=$preUpgradeCheckImage,prometheus.server.persistentVolume.enabled=false,prometheus.alertmanager.enabled=false,prometheus.kubeStateMetrics.enabled=false,prometheus.nodeExporter.enabled=false timeout 30 bash -c "helm_setup" diff --git a/test/upgrade/fission_upgrade_test.sh b/test/upgrade/fission_upgrade_test.sh index 1500c352..480aea81 100755 --- a/test/upgrade/fission_upgrade_test.sh +++ b/test/upgrade/fission_upgrade_test.sh @@ -93,7 +93,7 @@ sudo mv $ROOT/fission/fission /usr/local/bin/ ## Upgrade -helmVars=repository=$repo,image=$IMAGE,imageTag=$TAG,fetcherImage=$FETCHER_IMAGE,fetcherImageTag=$TAG,logger.fluentdImage=$FLUENTD_IMAGE,logger.fluentdImageTag=$TAG,functionNamespace=$fns,controllerPort=$controllerNodeport,pullPolicy=Always,analytics=false,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType +helmVars=repository=$repo,image=$IMAGE,imageTag=$TAG,fetcherImage=$FETCHER_IMAGE,fetcherImageTag=$TAG,logger.fluentdImageRepository=$repo,logger.fluentdImage=$FLUENTD_IMAGE,logger.fluentdImageTag=$TAG,functionNamespace=$fns,controllerPort=$controllerNodeport,pullPolicy=Always,analytics=false,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType echo "Upgrading fission" helm upgrade \ diff --git a/timer/main.go b/timer/main.go index 0186c211..8c9231dc 100644 --- a/timer/main.go +++ b/timer/main.go @@ -17,25 +17,26 @@ limitations under the License. package timer import ( - "log" + "github.com/pkg/errors" + "go.uber.org/zap" "github.com/fission/fission/crd" "github.com/fission/fission/publisher" ) -func Start(routerUrl string) error { +func Start(logger *zap.Logger, routerUrl string) error { fissionClient, _, _, err := crd.MakeFissionClient() if err != nil { - return err + return errors.Wrap(err, "failed to get fission or kubernetes client") } err = fissionClient.WaitForCRDs() if err != nil { - log.Fatalf("Error waiting for CRDs: %v", err) + return errors.Wrap(err, "error waiting for CRDs") } - poster := publisher.MakeWebhookPublisher(routerUrl) - MakeTimerSync(fissionClient, MakeTimer(poster)) + poster := publisher.MakeWebhookPublisher(logger, routerUrl) + MakeTimerSync(logger, fissionClient, MakeTimer(logger, poster)) return nil } diff --git a/timer/timer.go b/timer/timer.go index 16c35270..e9f2d940 100644 --- a/timer/timer.go +++ b/timer/timer.go @@ -17,9 +17,8 @@ limitations under the License. package timer import ( - "log" - "github.com/robfig/cron" + "go.uber.org/zap" "github.com/fission/fission" "github.com/fission/fission/crd" @@ -34,6 +33,7 @@ const ( type ( Timer struct { + logger *zap.Logger triggers map[string]*timerTriggerWithCron requestChannel chan *timerRequest publisher *publisher.Publisher @@ -53,8 +53,9 @@ type ( } ) -func MakeTimer(publisher publisher.Publisher) *Timer { +func MakeTimer(logger *zap.Logger, publisher publisher.Publisher) *Timer { timer := &Timer{ + logger: logger.Named("timer"), triggers: make(map[string]*timerTriggerWithCron), requestChannel: make(chan *timerRequest), publisher: &publisher, @@ -114,7 +115,7 @@ func (timer *Timer) syncCron(triggers []crd.TimeTrigger) error { if _, found := triggerMap[k]; !found { if v.cron != nil { v.cron.Stop() - log.Printf("Cron for time trigger %s stopped", v.trigger.Metadata.Name) + timer.logger.Info("cron for time trigger stopped", zap.String("trigger", v.trigger.Metadata.Name)) } delete(timer.triggers, k) } @@ -136,6 +137,6 @@ func (timer *Timer) newCron(t crd.TimeTrigger) *cron.Cron { (*timer.publisher).Publish("", headers, fission.UrlForFunction(t.Spec.FunctionReference.Name, t.Metadata.Namespace)) }) c.Start() - log.Printf("Add new cron for time trigger %v", t.Metadata.Name) + timer.logger.Info("added new cron for time trigger", zap.String("trigger", t.Metadata.Name)) return c } diff --git a/timer/timerSync.go b/timer/timerSync.go index c75ac9ea..c6d2fec5 100644 --- a/timer/timerSync.go +++ b/timer/timerSync.go @@ -17,9 +17,9 @@ limitations under the License. package timer import ( - "log" "time" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/fission/fission" @@ -28,13 +28,15 @@ import ( type ( TimerSync struct { + logger *zap.Logger fissionClient *crd.FissionClient timer *Timer } ) -func MakeTimerSync(fissionClient *crd.FissionClient, timer *Timer) *TimerSync { +func MakeTimerSync(logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync { ws := &TimerSync{ + logger: logger.Named("timer_sync"), fissionClient: fissionClient, timer: timer, } @@ -47,11 +49,11 @@ func (ws *TimerSync) syncSvc() { triggers, err := ws.fissionClient.TimeTriggers(metav1.NamespaceAll).List(metav1.ListOptions{}) if err != nil { if fission.IsNetworkError(err) { - log.Printf("Encounter network error, retry again: %v", err) + ws.logger.Info("encountered a network error - will retry", zap.Error(err)) time.Sleep(5 * time.Second) continue } - log.Fatalf("Failed get time trigger list: %v", err) + ws.logger.Fatal("failed to get time trigger list", zap.Error(err)) } ws.timer.Sync(triggers.Items)