Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use zap for logging (#1112)

Browse Source 
Use zap for logging
This commit is contained in:
Jon Carl
2019-03-14 21:11:15 +05:30
committed by Vishal
parent c717f182b6
commit 0fc864f230
98 changed files with 2031 additions and 1223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
builder/builder.go
+28 -25
View File
@@ -19,11 +19,9 @@ package builder
import ( import (
"bufio" "bufio"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"os" "os"
"os/exec" "os/exec"
@@ -33,6 +31,8 @@ import (
"time" "time"
"github.com/dchest/uniuri" "github.com/dchest/uniuri"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
) )
@@ -60,12 +60,14 @@ type (
} }
Builder struct { Builder struct {
logger *zap.Logger
sharedVolumePath string sharedVolumePath string
} }
) )
func MakeBuilder(sharedVolumePath string) *Builder { func MakeBuilder(logger *zap.Logger, sharedVolumePath string) *Builder {
return &Builder{ return &Builder{
logger: logger.Named("builder"),
sharedVolumePath: sharedVolumePath, sharedVolumePath: sharedVolumePath,
} }
} }
@@ -77,37 +79,37 @@ func (builder *Builder) VersionHandler(w http.ResponseWriter, r *http.Request) {
func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) { func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" { if r.Method != "POST" {
e := fmt.Sprintf("Method not allowed: %v", r.Method) e := "method not allowed"
log.Println(e) builder.logger.Error(e, zap.String("http_method", r.Method))
builder.reply(w, "", e, http.StatusMethodNotAllowed) builder.reply(w, "", fmt.Sprintf("%s: %s", e, r.Method), http.StatusMethodNotAllowed)
return return
} }
startTime := time.Now() startTime := time.Now()
defer func() { defer func() {
elapsed := time.Since(startTime) elapsed := time.Since(startTime)
log.Printf("elapsed time in build request = %v", elapsed) builder.logger.Info("build request complete", zap.Duration("elapsed_time", elapsed))
}() }()
// parse request // parse request
body, err := ioutil.ReadAll(r.Body) body, err := ioutil.ReadAll(r.Body)
if err != nil { if err != nil {
e := errors.New(fmt.Sprintf("Error reading request body: %v", err)) e := "error reading request body"
log.Println(e.Error()) builder.logger.Error(e, zap.Error(err))
builder.reply(w, "", e.Error(), http.StatusInternalServerError) builder.reply(w, "", fmt.Sprintf("%s: %s", e, err.Error()), http.StatusInternalServerError)
return return
} }
var req PackageBuildRequest var req PackageBuildRequest
err = json.Unmarshal(body, &req) err = json.Unmarshal(body, &req)
if err != nil { if err != nil {
e := errors.New(fmt.Sprintf("Error parsing json body: %v", err)) e := "error parsing json body"
log.Println(e.Error()) builder.logger.Error(e, zap.Error(err))
builder.reply(w, "", e.Error(), http.StatusBadRequest) builder.reply(w, "", fmt.Sprintf("%s: %s", e, err.Error()), http.StatusBadRequest)
return return
} }
log.Printf("Builder received request: %v", req) builder.logger.Info("builder received request", zap.Any("request", req))
log.Println("Starting build...") builder.logger.Info("starting build")
srcPkgPath := filepath.Join(builder.sharedVolumePath, req.SrcPkgFilename) srcPkgPath := filepath.Join(builder.sharedVolumePath, req.SrcPkgFilename)
deployPkgFilename := fmt.Sprintf("%v-%v", req.SrcPkgFilename, strings.ToLower(uniuri.NewLen(6))) deployPkgFilename := fmt.Sprintf("%v-%v", req.SrcPkgFilename, strings.ToLower(uniuri.NewLen(6)))
deployPkgPath := filepath.Join(builder.sharedVolumePath, deployPkgFilename) deployPkgPath := filepath.Join(builder.sharedVolumePath, deployPkgFilename)
@@ -118,10 +120,11 @@ func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) {
} }
buildLogs, err := builder.build(buildCmd, srcPkgPath, deployPkgPath) buildLogs, err := builder.build(buildCmd, srcPkgPath, deployPkgPath)
if err != nil { if err != nil {
e := errors.New(fmt.Sprintf("Error building source package: %v", err)) e := "error building source package"
log.Println(e.Error()) builder.logger.Error(e, zap.Error(err))
// append error at the end of build logs // append error at the end of build logs
buildLogs += fmt.Sprintf("%v\n", e.Error()) buildLogs += fmt.Sprintf("%s: %s\n", e, err.Error())
builder.reply(w, deployPkgFilename, buildLogs, http.StatusInternalServerError) builder.reply(w, deployPkgFilename, buildLogs, http.StatusInternalServerError)
return return
} }
@@ -137,7 +140,7 @@ func (builder *Builder) reply(w http.ResponseWriter, pkgFilename string, buildLo
rBody, err := json.Marshal(resp) rBody, err := json.Marshal(resp)
if err != nil { if err != nil {
e := errors.New(fmt.Sprintf("Error encoding response body: %v", err)) e := errors.Wrap(err, "error encoding response body")
rBody = []byte(fmt.Sprintf(`{"buildLogs": "%v"}`, e.Error())) rBody = []byte(fmt.Sprintf(`{"buildLogs": "%v"}`, e.Error()))
statusCode = http.StatusInternalServerError statusCode = http.StatusInternalServerError
} }
@@ -154,7 +157,7 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s
fi, err := os.Stat(srcPkgPath) fi, err := os.Stat(srcPkgPath)
if err != nil { if err != nil {
return "", errors.New(fmt.Sprintf("could not find srcPkgPath: '%s'", srcPkgPath)) return "", fmt.Errorf("could not find srcPkgPath: '%s'", srcPkgPath)
} }
if fi.IsDir() { if fi.IsDir() {
cmd.Dir = srcPkgPath cmd.Dir = srcPkgPath
@@ -170,12 +173,12 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s
stdout, err := cmd.StdoutPipe() stdout, err := cmd.StdoutPipe()
if err != nil { if err != nil {
return "", errors.New(fmt.Sprintf("Error creating stdout pipe for cmd: %v", err.Error())) return "", errors.Wrap(err, "error creating stdout pipe for cmd")
} }
stderr, err := cmd.StderrPipe() stderr, err := cmd.StderrPipe()
if err != nil { if err != nil {
return "", errors.New(fmt.Sprintf("Error creating stderr pipe for cmd: %v", err.Error())) return "", errors.Wrap(err, "error creating stderr pipe for cmd")
} }
var buildLogs string var buildLogs string
@@ -190,7 +193,7 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s
err = cmd.Start() err = cmd.Start()
if err != nil { if err != nil {
return "", errors.New(fmt.Sprintf("Error starting cmd: %v", err.Error())) return "", errors.Wrap(err, "error starting cmd")
} }
// Runtime logs // Runtime logs
@@ -201,14 +204,14 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s
} }
if err := scanner.Err(); err != nil { if err := scanner.Err(); err != nil {
scanErr := errors.New(fmt.Sprintf("Error reading cmd output: %v", err.Error())) scanErr := errors.Wrap(err, "error reading cmd output")
fmt.Println(scanErr) fmt.Println(scanErr)
return buildLogs, scanErr return buildLogs, scanErr
} }
err = cmd.Wait() err = cmd.Wait()
if err != nil { if err != nil {
cmdErr := errors.New(fmt.Sprintf("Error waiting for cmd '%v': %v", command, err.Error())) cmdErr := errors.Wrapf(err, "error waiting for cmd %q", command)
fmt.Println(cmdErr) fmt.Println(cmdErr)
return buildLogs, cmdErr return buildLogs, cmdErr
} }
builder/client/client.go
+12 -8
View File
@@ -20,31 +20,35 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"strings" "strings"
"time" "time"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
builder "github.com/fission/fission/builder" builder "github.com/fission/fission/builder"
) )
type ( type (
Client struct { Client struct {
url string logger *zap.Logger
url string
} }
) )
func MakeClient(builderUrl string) *Client { func MakeClient(logger *zap.Logger, builderUrl string) *Client {
return &Client{ return &Client{
url: strings.TrimSuffix(builderUrl, "/"), logger: logger.Named("builder_client"),
url: strings.TrimSuffix(builderUrl, "/"),
} }
} }
func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildResponse, error) { func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildResponse, error) {
body, err := json.Marshal(req) body, err := json.Marshal(req)
if err != nil { if err != nil {
return nil, err return nil, errors.Wrap(err, "error marshaling json")
} }
maxRetries := 20 maxRetries := 20
@@ -62,7 +66,7 @@ func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildR
if i < maxRetries-1 { if i < maxRetries-1 {
time.Sleep(50 * time.Duration(2*i) * time.Millisecond) time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error building package (%v), retrying", err) c.logger.Error("error building package, retrying", zap.Error(err))
continue continue
} }
@@ -73,14 +77,14 @@ func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildR
rBody, err := ioutil.ReadAll(resp.Body) rBody, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
log.Printf("Error reading resp body: %v", err) c.logger.Error("error reading resp body", zap.Error(err))
return nil, err return nil, err
} }
pkgBuildResp := builder.PackageBuildResponse{} pkgBuildResp := builder.PackageBuildResponse{}
err = json.Unmarshal([]byte(rBody), &pkgBuildResp) err = json.Unmarshal([]byte(rBody), &pkgBuildResp)
if err != nil { if err != nil {
log.Printf("Error parsing resp body: %v", err) c.logger.Error("error parsing resp body", zap.Error(err))
return nil, err return nil, err
} }
builder/cmd/main.go
+10 -2
View File
@@ -21,21 +21,29 @@ import (
"net/http" "net/http"
"os" "os"
"go.uber.org/zap"
builder "github.com/fission/fission/builder" builder "github.com/fission/fission/builder"
) )
// Usage: builder <shared volume path> // Usage: builder <shared volume path>
func main() { func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
dir := os.Args[1] dir := os.Args[1]
if _, err := os.Stat(dir); err != nil { if _, err := os.Stat(dir); err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
err = os.MkdirAll(dir, os.ModeDir|0700) err = os.MkdirAll(dir, os.ModeDir|0700)
if err != nil { if err != nil {
log.Fatalf("Error creating directory: %v", err) logger.Fatal("error creating directory", zap.Error(err), zap.String("directory", dir))
} }
} }
} }
builder := builder.MakeBuilder(dir) builder := builder.MakeBuilder(logger, dir)
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/", builder.Handler) mux.HandleFunc("/", builder.Handler)
mux.HandleFunc("/version", builder.VersionHandler) mux.HandleFunc("/version", builder.VersionHandler)
buildermgr/buildermgr.go
+8 -7
View File
@@ -17,29 +17,30 @@ limitations under the License.
package buildermgr package buildermgr
import ( import (
"log" "github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
) )
// Start the buildermgr service. // Start the buildermgr service.
func Start(storageSvcUrl string, envBuilderNamespace string) error { func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
bmLogger := logger.Named("builder_manager")
fissionClient, kubernetesClient, _, err := crd.MakeFissionClient() fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
if err != nil { if err != nil {
log.Printf("Failed to get kubernetes client: %v", err) return errors.Wrap(err, "failed to get fission or kubernetes client")
return err
} }
err = fissionClient.WaitForCRDs() err = fissionClient.WaitForCRDs()
if err != nil { if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err) return errors.Wrap(err, "error waiting for CRDs")
} }
envWatcher := makeEnvironmentWatcher(fissionClient, kubernetesClient, envBuilderNamespace) envWatcher := makeEnvironmentWatcher(bmLogger, fissionClient, kubernetesClient, envBuilderNamespace)
go envWatcher.watchEnvironments() go envWatcher.watchEnvironments()
pkgWatcher := makePackageWatcher(fissionClient, pkgWatcher := makePackageWatcher(bmLogger, fissionClient,
kubernetesClient, envBuilderNamespace, storageSvcUrl) kubernetesClient, envBuilderNamespace, storageSvcUrl)
go pkgWatcher.watchPackages(fissionClient, kubernetesClient, envBuilderNamespace) go pkgWatcher.watchPackages(fissionClient, kubernetesClient, envBuilderNamespace)
buildermgr/common.go
+19 -16
View File
@@ -19,11 +19,13 @@ package buildermgr
import ( import (
"context" "context"
"fmt" "fmt"
"log"
"net/http" "net/http"
"strings" "strings"
"github.com/dchest/uniuri" "github.com/dchest/uniuri"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/builder" "github.com/fission/fission/builder"
builderClient "github.com/fission/fission/builder/client" builderClient "github.com/fission/fission/builder/client"
@@ -38,20 +40,21 @@ import (
// 3. Send upload request to fetcher to upload deployment package. // 3. Send upload request to fetcher to upload deployment package.
// 4. Return upload response and build logs. // 4. Return upload response and build logs.
// *. Return build logs and error if any one of steps above failed. // *. Return build logs and error if any one of steps above failed.
func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuilderNamespace string, func buildPackage(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, envBuilderNamespace string,
storageSvcUrl string, pkg *crd.Package) (uploadResp *fission.ArchiveUploadResponse, buildLogs string, err error) { storageSvcUrl string, pkg *crd.Package) (uploadResp *fission.ArchiveUploadResponse, buildLogs string, err error) {
env, err := fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name) env, err := fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
if err != nil { if err != nil {
e := fmt.Sprintf("Error getting environment CRD info: %v", err) e := "error getting environment CRD info"
log.Println(e) logger.Error(e, zap.Error(err))
e = fmt.Sprintf("%s: %v", e, err)
return nil, e, fission.MakeError(http.StatusInternalServerError, e) return nil, e, fission.MakeError(http.StatusInternalServerError, e)
} }
svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, envBuilderNamespace) svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, envBuilderNamespace)
srcPkgFilename := fmt.Sprintf("%v-%v", pkg.Metadata.Name, strings.ToLower(uniuri.NewLen(6))) srcPkgFilename := fmt.Sprintf("%v-%v", pkg.Metadata.Name, strings.ToLower(uniuri.NewLen(6)))
fetcherC := fetcherClient.MakeClient(fmt.Sprintf("http://%v:8000", svcName)) fetcherC := fetcherClient.MakeClient(logger, fmt.Sprintf("http://%v:8000", svcName))
builderC := builderClient.MakeClient(fmt.Sprintf("http://%v:8001", svcName)) builderC := builderClient.MakeClient(logger, fmt.Sprintf("http://%v:8001", svcName))
fetchReq := &fission.FunctionFetchRequest{ fetchReq := &fission.FunctionFetchRequest{
FetchType: fission.FETCH_SOURCE, FetchType: fission.FETCH_SOURCE,
@@ -63,8 +66,9 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
// send fetch request to fetcher // send fetch request to fetcher
err = fetcherC.Fetch(ctx, fetchReq) err = fetcherC.Fetch(ctx, fetchReq)
if err != nil { if err != nil {
e := fmt.Sprintf("Error fetching source package: %v", err) e := "error fetching source package"
log.Println(e) logger.Error(e, zap.Error(err))
e = fmt.Sprintf("%s: %v", e, err)
return nil, e, fission.MakeError(http.StatusInternalServerError, e) return nil, e, fission.MakeError(http.StatusInternalServerError, e)
} }
@@ -78,12 +82,11 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
BuildCommand: buildCmd, BuildCommand: buildCmd,
} }
log.Printf("Start building with source package: %v", srcPkgFilename) logger.Info("started building with source package", zap.String("source_package", srcPkgFilename))
// send build request to builder // send build request to builder
buildResp, err := builderC.Build(pkgBuildReq) buildResp, err := builderC.Build(pkgBuildReq)
if err != nil { if err != nil {
e := fmt.Sprintf("Error building deployment package: %v", err) e := fmt.Sprintf("Error building deployment package: %v", err)
log.Println(e)
var buildLogs string var buildLogs string
if buildResp != nil { if buildResp != nil {
buildLogs = buildResp.BuildLogs buildLogs = buildResp.BuildLogs
@@ -92,7 +95,7 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
return nil, buildLogs, fission.MakeError(http.StatusInternalServerError, e) return nil, buildLogs, fission.MakeError(http.StatusInternalServerError, e)
} }
log.Printf("Build succeed, source package: %v, deployment package: %v", srcPkgFilename, buildResp.ArtifactFilename) logger.Info("build succeed", zap.String("source_package", srcPkgFilename), zap.String("deployment_package", buildResp.ArtifactFilename))
archivePackage := !env.Spec.KeepArchive archivePackage := !env.Spec.KeepArchive
@@ -102,12 +105,11 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
ArchivePackage: archivePackage, ArchivePackage: archivePackage,
} }
log.Printf("Start uploading deployment package: %v", buildResp.ArtifactFilename) logger.Info("started uploading deployment package", zap.String("deployment_package", buildResp.ArtifactFilename))
// ask fetcher to upload the deployment package // ask fetcher to upload the deployment package
uploadResp, err = fetcherC.Upload(ctx, uploadReq) uploadResp, err = fetcherC.Upload(ctx, uploadReq)
if err != nil { if err != nil {
e := fmt.Sprintf("Error uploading deployment package: %v", err) e := fmt.Sprintf("Error uploading deployment package: %v", err)
log.Println(e)
buildResp.BuildLogs += fmt.Sprintf("%v\n", e) buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
return nil, buildResp.BuildLogs, fission.MakeError(http.StatusInternalServerError, e) return nil, buildResp.BuildLogs, fission.MakeError(http.StatusInternalServerError, e)
} }
@@ -115,7 +117,7 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
return uploadResp, buildResp.BuildLogs, nil return uploadResp, buildResp.BuildLogs, nil
} }
func updatePackage(fissionClient *crd.FissionClient, func updatePackage(logger *zap.Logger, fissionClient *crd.FissionClient,
pkg *crd.Package, status fission.BuildStatus, buildLogs string, pkg *crd.Package, status fission.BuildStatus, buildLogs string,
uploadResp *fission.ArchiveUploadResponse) (*crd.Package, error) { uploadResp *fission.ArchiveUploadResponse) (*crd.Package, error) {
@@ -135,8 +137,9 @@ func updatePackage(fissionClient *crd.FissionClient,
// update package spec // update package spec
pkg, err := fissionClient.Packages(pkg.Metadata.Namespace).Update(pkg) pkg, err := fissionClient.Packages(pkg.Metadata.Namespace).Update(pkg)
if err != nil { if err != nil {
log.Printf("Error updating package: %v", err) e := "error updating package"
return nil, err logger.Error(e, zap.Error(err))
return nil, errors.Wrap(err, e)
} }
// return resource version for function to update function package ref // return resource version for function to update function package ref
buildermgr/envwatcher.go
+34 -28
View File
@@ -18,11 +18,12 @@ package buildermgr
import ( import (
"fmt" "fmt"
"log"
"os" "os"
"strconv" "strconv"
"time" "time"
"github.com/pkg/errors"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1" "k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -73,6 +74,7 @@ type (
} }
environmentWatcher struct { environmentWatcher struct {
logger *zap.Logger
cache map[string]*builderInfo cache map[string]*builderInfo
requestChan chan envwRequest requestChan chan envwRequest
builderNamespace string builderNamespace string
@@ -86,7 +88,7 @@ type (
} }
) )
func makeEnvironmentWatcher(fissionClient *crd.FissionClient, func makeEnvironmentWatcher(logger *zap.Logger, fissionClient *crd.FissionClient,
kubernetesClient *kubernetes.Clientset, builderNamespace string) *environmentWatcher { kubernetesClient *kubernetes.Clientset, builderNamespace string) *environmentWatcher {
useIstio := false useIstio := false
@@ -94,7 +96,7 @@ func makeEnvironmentWatcher(fissionClient *crd.FissionClient,
if len(enableIstio) > 0 { if len(enableIstio) > 0 {
istio, err := strconv.ParseBool(enableIstio) istio, err := strconv.ParseBool(enableIstio)
if err != nil { if err != nil {
log.Println("Failed to parse ENABLE_ISTIO, defaults to false") logger.Info("Failed to parse ENABLE_ISTIO, defaults to false")
} }
useIstio = istio useIstio = istio
} }
@@ -109,6 +111,7 @@ func makeEnvironmentWatcher(fissionClient *crd.FissionClient,
collectorEndpoint := os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT") collectorEndpoint := os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT")
envWatcher := &environmentWatcher{ envWatcher := &environmentWatcher{
logger: logger.Named("environment_watcher"),
cache: make(map[string]*builderInfo), cache: make(map[string]*builderInfo),
requestChan: make(chan envwRequest), requestChan: make(chan envwRequest),
builderNamespace: builderNamespace, builderNamespace: builderNamespace,
@@ -153,11 +156,11 @@ func (envw *environmentWatcher) watchEnvironments() {
}) })
if err != nil { if err != nil {
if fission.IsNetworkError(err) { if fission.IsNetworkError(err) {
log.Printf("Encounter network error, retrying later: %v", err) envw.logger.Error("encountered network error, retrying later", zap.Error(err))
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
continue continue
} }
log.Fatalf("Error watching environment list: %v", err) envw.logger.Fatal("error watching environment list", zap.Error(err))
} }
for { for {
@@ -185,11 +188,11 @@ func (envw *environmentWatcher) sync() {
envList, err := envw.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{}) envList, err := envw.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil { if err != nil {
if fission.IsNetworkError(err) { if fission.IsNetworkError(err) {
log.Printf("Error syncing environment CRD resources due to network error, retrying later: %v", err) envw.logger.Error("error syncing environment CRD resources due to network error, retrying later", zap.Error(err))
time.Sleep(50 * time.Duration(2*i) * time.Millisecond) time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
continue continue
} }
log.Fatalf("Error syncing environment CRD resources: %v", err) envw.logger.Fatal("error syncing environment CRD resources", zap.Error(err))
} }
// Create environment builders for all environments // Create environment builders for all environments
@@ -202,7 +205,7 @@ func (envw *environmentWatcher) sync() {
} }
_, err := envw.getEnvBuilder(&env) _, err := envw.getEnvBuilder(&env)
if err != nil { if err != nil {
log.Printf("Error creating builder for %v: %v", env.Metadata.Name, err) envw.logger.Error("error creating builder", zap.Error(err), zap.String("builder_target", env.Metadata.Name))
} }
} }
@@ -258,7 +261,7 @@ func (envw *environmentWatcher) service() {
svcList, err := envw.getBuilderServiceList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll) svcList, err := envw.getBuilderServiceList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
if err != nil { if err != nil {
log.Println(err.Error()) envw.logger.Error("error getting the builder service list", zap.Error(err))
} }
for _, svc := range svcList { for _, svc := range svcList {
envName := svc.ObjectMeta.Labels[LABEL_ENV_NAME] envName := svc.ObjectMeta.Labels[LABEL_ENV_NAME]
@@ -268,7 +271,9 @@ func (envw *environmentWatcher) service() {
if _, ok := latestEnvList[key]; !ok { if _, ok := latestEnvList[key]; !ok {
err := envw.deleteBuilderServiceByName(svc.ObjectMeta.Name, svc.ObjectMeta.Namespace) err := envw.deleteBuilderServiceByName(svc.ObjectMeta.Name, svc.ObjectMeta.Namespace)
if err != nil { if err != nil {
log.Printf("Error removing builder service: %v", err) envw.logger.Error("error removing builder service", zap.Error(err),
zap.String("service_name", svc.ObjectMeta.Name),
zap.String("service_namespace", svc.ObjectMeta.Namespace))
} }
} }
delete(envw.cache, key) delete(envw.cache, key)
@@ -276,7 +281,7 @@ func (envw *environmentWatcher) service() {
deployList, err := envw.getBuilderDeploymentList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll) deployList, err := envw.getBuilderDeploymentList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
if err != nil { if err != nil {
log.Printf(err.Error()) envw.logger.Error("error getting the builder deployment list", zap.Error(err))
} }
for _, deploy := range deployList { for _, deploy := range deployList {
envName := deploy.ObjectMeta.Labels[LABEL_ENV_NAME] envName := deploy.ObjectMeta.Labels[LABEL_ENV_NAME]
@@ -286,7 +291,9 @@ func (envw *environmentWatcher) service() {
if _, ok := latestEnvList[key]; !ok { if _, ok := latestEnvList[key]; !ok {
err := envw.deleteBuilderDeploymentByName(deploy.ObjectMeta.Name, deploy.ObjectMeta.Namespace) err := envw.deleteBuilderDeploymentByName(deploy.ObjectMeta.Name, deploy.ObjectMeta.Namespace)
if err != nil { if err != nil {
log.Printf("Error removing builder deployment: %v", err) envw.logger.Error("error removing builder deployment", zap.Error(err),
zap.String("deployment_name", deploy.ObjectMeta.Name),
zap.String("deployment_namespace", deploy.ObjectMeta.Namespace))
} }
} }
delete(envw.cache, key) delete(envw.cache, key)
@@ -327,12 +334,12 @@ func (envw *environmentWatcher) createBuilder(env *crd.Environment, ns string) (
if len(svcList) == 0 { if len(svcList) == 0 {
svc, err = envw.createBuilderService(env, ns) svc, err = envw.createBuilderService(env, ns)
if err != nil { if err != nil {
return nil, fmt.Errorf("Error creating builder service: %v", err) return nil, errors.Wrap(err, "error creating builder service")
} }
} else if len(svcList) == 1 { } else if len(svcList) == 1 {
svc = &svcList[0] svc = &svcList[0]
} else { } else {
return nil, fmt.Errorf("Found more than one builder service for environment %v", env.Metadata.Name) return nil, fmt.Errorf("found more than one builder service for environment %q", env.Metadata.Name)
} }
deployList, err := envw.getBuilderDeploymentList(sel, ns) deployList, err := envw.getBuilderDeploymentList(sel, ns)
@@ -344,18 +351,17 @@ func (envw *environmentWatcher) createBuilder(env *crd.Environment, ns string) (
// create builder SA in this ns, if not already created // create builder SA in this ns, if not already created
_, err := fission.SetupSA(envw.kubernetesClient, fission.FissionBuilderSA, ns) _, err := fission.SetupSA(envw.kubernetesClient, fission.FissionBuilderSA, ns)
if err != nil { if err != nil {
log.Printf("Error : %v creating %s in ns : %s", err, fission.FissionBuilderSA, ns) return nil, errors.Wrapf(err, "error creating %q in ns: %s", fission.FissionBuilderSA, ns)
return nil, err
} }
deploy, err = envw.createBuilderDeployment(env, ns) deploy, err = envw.createBuilderDeployment(env, ns)
if err != nil { if err != nil {
return nil, fmt.Errorf("Error creating builder deployment: %v", err) return nil, errors.Wrap(err, "error creating builder deployment")
} }
} else if len(deployList) == 1 { } else if len(deployList) == 1 {
deploy = &deployList[0] deploy = &deployList[0]
} else { } else {
return nil, fmt.Errorf("Found more than one builder deployment for environment %v", env.Metadata.Name) return nil, fmt.Errorf("found more than one builder deployment for environment %q", env.Metadata.Name)
} }
return &builderInfo{ return &builderInfo{
@@ -370,7 +376,7 @@ func (envw *environmentWatcher) deleteBuilderServiceByName(name, namespace strin
Services(namespace). Services(namespace).
Delete(name, &delOpt) Delete(name, &delOpt)
if err != nil { if err != nil {
return fmt.Errorf("Error deleting builder service %s.%s: %v", name, namespace, err) return errors.Wrapf(err, "error deleting builder service %s.%s", name, namespace)
} }
return nil return nil
} }
@@ -380,7 +386,7 @@ func (envw *environmentWatcher) deleteBuilderDeploymentByName(name, namespace st
Deployments(namespace). Deployments(namespace).
Delete(name, &delOpt) Delete(name, &delOpt)
if err != nil { if err != nil {
return fmt.Errorf("Error deleting builder deployment %s.%s: %v", name, namespace, err) return errors.Wrapf(err, "error deleting builder deployment %s.%s", name, namespace)
} }
return nil return nil
} }
@@ -391,12 +397,12 @@ func (envw *environmentWatcher) deleteBuilderService(sel map[string]string, ns s
return err return err
} }
for _, svc := range svcList { for _, svc := range svcList {
log.Printf("Removing builder service: %v", svc.ObjectMeta.Name) envw.logger.Info("removing builder service", zap.String("service_name", svc.ObjectMeta.Name))
err = envw.kubernetesClient.CoreV1(). err = envw.kubernetesClient.CoreV1().
Services(ns). Services(ns).
Delete(svc.ObjectMeta.Name, &delOpt) Delete(svc.ObjectMeta.Name, &delOpt)
if err != nil { if err != nil {
return fmt.Errorf("Error deleting builder service: %v", err) return errors.Wrap(err, "error deleting builder service")
} }
} }
return nil return nil
@@ -408,12 +414,12 @@ func (envw *environmentWatcher) deleteBuilderDeployment(sel map[string]string, n
return err return err
} }
for _, deploy := range deployList { for _, deploy := range deployList {
log.Printf("Removing builder deployment: %v", deploy.ObjectMeta.Name) envw.logger.Info("removing builder deployment", zap.String("deployment_name", deploy.ObjectMeta.Name))
err = envw.kubernetesClient.ExtensionsV1beta1(). err = envw.kubernetesClient.ExtensionsV1beta1().
Deployments(ns). Deployments(ns).
Delete(deploy.ObjectMeta.Name, &delOpt) Delete(deploy.ObjectMeta.Name, &delOpt)
if err != nil { if err != nil {
return fmt.Errorf("Error deleteing builder deployment: %v", err) return errors.Wrap(err, "error deleteing builder deployment")
} }
} }
return nil return nil
@@ -425,7 +431,7 @@ func (envw *environmentWatcher) getBuilderServiceList(sel map[string]string, ns
LabelSelector: labels.Set(sel).AsSelector().String(), LabelSelector: labels.Set(sel).AsSelector().String(),
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("Error getting builder service list: %v", err) return nil, errors.Wrap(err, "error getting builder service list")
} }
return svcList.Items, nil return svcList.Items, nil
} }
@@ -464,7 +470,7 @@ func (envw *environmentWatcher) createBuilderService(env *crd.Environment, ns st
}, },
}, },
} }
log.Printf("Creating builder service: %v", name) envw.logger.Info("creating builder service", zap.String("service_name", name))
_, err := envw.kubernetesClient.CoreV1().Services(ns).Create(&service) _, err := envw.kubernetesClient.CoreV1().Services(ns).Create(&service)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -478,7 +484,7 @@ func (envw *environmentWatcher) getBuilderDeploymentList(sel map[string]string,
LabelSelector: labels.Set(sel).AsSelector().String(), LabelSelector: labels.Set(sel).AsSelector().String(),
}) })
if err != nil { if err != nil {
return nil, fmt.Errorf("Error getting builder deployment list: %v", err) return nil, errors.Wrap(err, "error getting builder deployment list")
} }
return deployList.Items, nil return deployList.Items, nil
} }
@@ -615,7 +621,7 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns
}, },
}, },
} }
log.Printf("Creating builder deployment: %v", fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion)) envw.logger.Info("creating builder deployment", zap.String("deployment", name))
_, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Create(deployment) _, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Create(deployment)
if err != nil { if err != nil {
return nil, err return nil, err
buildermgr/pkgwatcher.go
+45 -37
View File
@@ -19,23 +19,24 @@ package buildermgr
import ( import (
"context" "context"
"fmt" "fmt"
"log"
"time" "time"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/fields"
"k8s.io/client-go/kubernetes"
k8sCache "k8s.io/client-go/tools/cache" k8sCache "k8s.io/client-go/tools/cache"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/cache" "github.com/fission/fission/cache"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
"k8s.io/client-go/kubernetes"
) )
type ( type (
packageWatcher struct { packageWatcher struct {
logger *zap.Logger
fissionClient *crd.FissionClient fissionClient *crd.FissionClient
k8sClient *kubernetes.Clientset k8sClient *kubernetes.Clientset
podStore k8sCache.Store podStore k8sCache.Store
@@ -45,13 +46,14 @@ type (
} }
) )
func makePackageWatcher(fissionClient *crd.FissionClient, k8sClientSet *kubernetes.Clientset, func makePackageWatcher(logger *zap.Logger, fissionClient *crd.FissionClient, k8sClientSet *kubernetes.Clientset,
builderNamespace string, storageSvcUrl string) *packageWatcher { builderNamespace string, storageSvcUrl string) *packageWatcher {
lw := k8sCache.NewListWatchFromClient(k8sClientSet.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.Everything()) lw := k8sCache.NewListWatchFromClient(k8sClientSet.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.Everything())
store, controller := k8sCache.NewInformer(lw, &apiv1.Pod{}, 30*time.Second, k8sCache.ResourceEventHandlerFuncs{}) store, controller := k8sCache.NewInformer(lw, &apiv1.Pod{}, 30*time.Second, k8sCache.ResourceEventHandlerFuncs{})
go controller.Run(make(chan struct{})) go controller.Run(make(chan struct{}))
pkgw := &packageWatcher{ pkgw := &packageWatcher{
logger: logger.Named("package_watcher"),
fissionClient: fissionClient, fissionClient: fissionClient,
k8sClient: k8sClientSet, k8sClient: k8sClientSet,
podStore: store, podStore: store,
@@ -86,20 +88,20 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
} }
defer buildCache.Delete(key) defer buildCache.Delete(key)
log.Printf("Start build for package %v with resource version %v", srcpkg.Metadata.Name, srcpkg.Metadata.ResourceVersion) pkgw.logger.Info("starting build for package", zap.String("package_name", srcpkg.Metadata.Name), zap.String("resource_version", srcpkg.Metadata.ResourceVersion))
pkg, err := updatePackage(pkgw.fissionClient, srcpkg, fission.BuildStatusRunning, "", nil) pkg, err := updatePackage(pkgw.logger, pkgw.fissionClient, srcpkg, fission.BuildStatusRunning, "", nil)
if err != nil { if err != nil {
e := fmt.Sprintf("Error setting package pending state: %v", err) pkgw.logger.Error("error setting package pending state", zap.Error(err))
log.Println(e)
updatePackage(pkgw.fissionClient, srcpkg, fission.BuildStatusFailed, e, nil)
return return
} }
env, err := pkgw.fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name) env, err := pkgw.fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
if errors.IsNotFound(err) { if k8serrors.IsNotFound(err) {
updatePackage(pkgw.fissionClient, pkg, e := "environment does not exist"
fission.BuildStatusFailed, "Environment not existed", nil) pkgw.logger.Error(e, zap.String("environment", pkg.Spec.Environment.Name))
updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
fission.BuildStatusFailed, fmt.Sprintf("%s: %q", e, pkg.Spec.Environment.Name), nil)
return return
} }
@@ -109,12 +111,12 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
// iterate all available environment builders. // iterate all available environment builders.
items := pkgw.podStore.List() items := pkgw.podStore.List()
if err != nil { if err != nil {
log.Printf("Error retrieving pod information for env %v: %v", err, env.Metadata.Name) pkgw.logger.Error("error retrieving pod information for environment", zap.Error(err), zap.String("environment", env.Metadata.Name))
return return
} }
if len(items) == 0 { if len(items) == 0 {
log.Printf("Environment \"%v\" builder pod is not existed yet, retry again later.", pkg.Spec.Environment.Name) pkgw.logger.Info("builder pod does not exist for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name))
time.Sleep(time.Duration(i*1) * time.Second) time.Sleep(time.Duration(i*1) * time.Second)
continue continue
} }
@@ -145,7 +147,7 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
} }
if !podIsReady { if !podIsReady {
log.Printf("Environment \"%v\" builder pod is not ready, retry again later.", pkg.Spec.Environment.Name) pkgw.logger.Info("builder pod is not ready for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name))
time.Sleep(time.Duration(i*1) * time.Second) time.Sleep(time.Duration(i*1) * time.Second)
break break
} }
@@ -153,31 +155,37 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
// Add the package getter rolebinding to builder sa // Add the package getter rolebinding to builder sa
// we continue here if role binding was not setup succeesffully. this is because without this, the fetcher wont be able to fetch the source pkg into the container and // we continue here if role binding was not setup succeesffully. this is because without this, the fetcher wont be able to fetch the source pkg into the container and
// the build will fail eventually // the build will fail eventually
err := fission.SetupRoleBinding(pkgw.k8sClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, builderNs) err := fission.SetupRoleBinding(pkgw.logger, pkgw.k8sClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, builderNs)
if err != nil { if err != nil {
log.Printf("Error : %v in setting up the role binding %s for pkg : %s.%s", err, fission.PackageGetterRB, pkg.Metadata.Name, pkg.Metadata.Namespace) pkgw.logger.Error("error setting up role binding for package",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namespace", pkg.Metadata.Namespace))
continue continue
} else { } else {
log.Printf("Setup rolebinding for sa : %s.%s for pkg : %s.%s", fission.FissionBuilderSA, builderNs, pkg.Metadata.Name, pkg.Metadata.Namespace) pkgw.logger.Info("setup rolebinding for sa package",
zap.String("sa", fmt.Sprintf("%s.%s", fission.FissionBuilderSA, builderNs)),
zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)))
} }
ctx := context.Background() ctx := context.Background()
uploadResp, buildLogs, err := buildPackage(ctx, pkgw.fissionClient, builderNs, pkgw.storageSvcUrl, pkg) uploadResp, buildLogs, err := buildPackage(ctx, pkgw.logger, pkgw.fissionClient, builderNs, pkgw.storageSvcUrl, pkg)
if err != nil { if err != nil {
log.Printf("Error building package %v: %v", pkg.Metadata.Name, err) pkgw.logger.Error("error building package", zap.Error(err), zap.String("package_name", pkg.Metadata.Name))
updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
return return
} }
log.Printf("Start updating info of package: %v", pkg.Metadata.Name) pkgw.logger.Info("starting package info update", zap.String("package_name", pkg.Metadata.Name))
fnList, err := pkgw.fissionClient. fnList, err := pkgw.fissionClient.
Functions(metav1.NamespaceAll).List(metav1.ListOptions{}) Functions(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil { if err != nil {
e := fmt.Sprintf("Error getting function list: %v", err) e := "error getting function list"
log.Println(e) pkgw.logger.Error(e, zap.Error(err))
buildLogs += fmt.Sprintf("%v\n", e) buildLogs += fmt.Sprintf("%s: %v\n", e, err)
updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
} }
// A package may be used by multiple functions. Update // A package may be used by multiple functions. Update
@@ -190,33 +198,33 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
// update CRD // update CRD
_, err = pkgw.fissionClient.Functions(fn.Metadata.Namespace).Update(&fn) _, err = pkgw.fissionClient.Functions(fn.Metadata.Namespace).Update(&fn)
if err != nil { if err != nil {
e := fmt.Sprintf("Error updating function package resource version: %v", err) e := "error updating function package resource version"
log.Println(e) pkgw.logger.Error(e, zap.Error(err))
buildLogs += fmt.Sprintf("%v\n", e) buildLogs += fmt.Sprintf("%s: %v\n", e, err)
updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
return return
} }
} }
} }
_, err = updatePackage(pkgw.fissionClient, pkg, _, err = updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
fission.BuildStatusSucceeded, buildLogs, uploadResp) fission.BuildStatusSucceeded, buildLogs, uploadResp)
if err != nil { if err != nil {
log.Printf("Error update package info: %v", err) pkgw.logger.Error("error updating package info", zap.Error(err), zap.String("package_name", pkg.Metadata.Name))
updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil) updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
return return
} }
log.Printf("Completed build request for package: %v", pkg.Metadata.Name) pkgw.logger.Info("completed package build request", zap.String("package_name", pkg.Metadata.Name))
return return
} }
} }
// build timeout // build timeout
updatePackage(pkgw.fissionClient, pkg, updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
fission.BuildStatusFailed, "Build timeout due to environment builder not ready", nil) fission.BuildStatusFailed, "Build timeout due to environment builder not ready", nil)
log.Printf("Max retries exceeded in building the source pkg : %s.%s, timeout due to environment builder not ready", pkgw.logger.Error("max retries exceeded in building source package, timeout due to environment builder not ready",
pkg.Metadata.Name, pkg.Metadata.Namespace) zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)))
return return
} }
canaryconfigmgr/canaryConfigCache.go
+1 -1
View File
@@ -18,11 +18,11 @@ package canaryconfigmgr
import ( import (
"context" "context"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/cache" "github.com/fission/fission/cache"
"time"
) )
type ( type (
canaryconfigmgr/canaryConfigMgr.go
+163 -46
View File
@@ -24,7 +24,8 @@ import (
"strings" "strings"
"time" "time"
log "github.com/sirupsen/logrus" "github.com/pkg/errors"
"go.uber.org/zap"
k8serrors "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/fields"
@@ -37,6 +38,7 @@ import (
) )
type canaryConfigMgr struct { type canaryConfigMgr struct {
logger *zap.Logger
fissionClient *crd.FissionClient fissionClient *crd.FissionClient
kubeClient *kubernetes.Clientset kubeClient *kubernetes.Clientset
canaryConfigStore k8sCache.Store canaryConfigStore k8sCache.Store
@@ -46,9 +48,9 @@ type canaryConfigMgr struct {
canaryCfgCancelFuncMap *canaryConfigCancelFuncMap canaryCfgCancelFuncMap *canaryConfigCancelFuncMap
} }
func MakeCanaryConfigMgr(fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, crdClient *rest.RESTClient, prometheusSvc string) (*canaryConfigMgr, error) { func MakeCanaryConfigMgr(logger *zap.Logger, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, crdClient *rest.RESTClient, prometheusSvc string) (*canaryConfigMgr, error) {
if prometheusSvc == "" { if prometheusSvc == "" {
log.Info("Try to retrieve prometheus server information from environment variables") logger.Info("try to retrieve prometheus server information from environment variables")
var prometheusSvcHost, prometheusSvcPort string var prometheusSvcHost, prometheusSvcPort string
// handle a case where there is a prometheus server is already installed, try to find the service from env variable // handle a case where there is a prometheus server is already installed, try to find the service from env variable
@@ -73,12 +75,13 @@ func MakeCanaryConfigMgr(fissionClient *crd.FissionClient, kubeClient *kubernete
return nil, fmt.Errorf("prometheus service url not found/invalid, cant create canary config manager: %v", prometheusSvc) return nil, fmt.Errorf("prometheus service url not found/invalid, cant create canary config manager: %v", prometheusSvc)
} }
promClient, err := MakePrometheusClient(prometheusSvc) promClient, err := MakePrometheusClient(logger, prometheusSvc)
if err != nil { if err != nil {
return nil, err return nil, err
} }
configMgr := &canaryConfigMgr{ configMgr := &canaryConfigMgr{
logger: logger.Named("canary_config_manager"),
fissionClient: fissionClient, fissionClient: fissionClient,
kubeClient: kubeClient, kubeClient: kubeClient,
crdClient: crdClient, crdClient: crdClient,
@@ -113,7 +116,10 @@ func (canaryCfgMgr *canaryConfigMgr) initCanaryConfigController() (k8sCache.Stor
newConfig := newObj.(*crd.CanaryConfig) newConfig := newObj.(*crd.CanaryConfig)
if oldConfig.Metadata.ResourceVersion != newConfig.Metadata.ResourceVersion && if oldConfig.Metadata.ResourceVersion != newConfig.Metadata.ResourceVersion &&
newConfig.Status.Status == fission.CanaryConfigStatusPending { newConfig.Status.Status == fission.CanaryConfigStatusPending {
log.Printf("update canary config invoked for : %s.%s, newConfig.Status.Status=%s", newConfig.Metadata.Name, newConfig.Metadata.Namespace, newConfig.Status.Status) canaryCfgMgr.logger.Info("update canary config invoked",
zap.String("name", newConfig.Metadata.Name),
zap.String("namespace", newConfig.Metadata.Namespace),
zap.String("version", newConfig.Metadata.ResourceVersion))
go canaryCfgMgr.updateCanaryConfig(oldConfig, newConfig) go canaryCfgMgr.updateCanaryConfig(oldConfig, newConfig)
} }
go canaryCfgMgr.reSyncCanaryConfigs() go canaryCfgMgr.reSyncCanaryConfigs()
@@ -126,17 +132,21 @@ func (canaryCfgMgr *canaryConfigMgr) initCanaryConfigController() (k8sCache.Stor
func (canaryCfgMgr *canaryConfigMgr) Run(ctx context.Context) { func (canaryCfgMgr *canaryConfigMgr) Run(ctx context.Context) {
go canaryCfgMgr.canaryConfigController.Run(ctx.Done()) go canaryCfgMgr.canaryConfigController.Run(ctx.Done())
log.Printf("started Canary configmgr controller") canaryCfgMgr.logger.Info("started canary configmgr controller")
} }
func (canaryCfgMgr *canaryConfigMgr) addCanaryConfig(canaryConfig *crd.CanaryConfig) { func (canaryCfgMgr *canaryConfigMgr) addCanaryConfig(canaryConfig *crd.CanaryConfig) {
log.Printf("addCanaryConfig called for %s", canaryConfig.Metadata.Name) canaryCfgMgr.logger.Info("addCanaryConfig called", zap.String("canary_config", canaryConfig.Metadata.Name))
// for each canary config, create a ticker with increment interval // for each canary config, create a ticker with increment interval
interval, err := time.ParseDuration(canaryConfig.Spec.WeightIncrementDuration) interval, err := time.ParseDuration(canaryConfig.Spec.WeightIncrementDuration)
if err != nil { if err != nil {
log.Printf("Error parsing duration: %v, cant proceed with this canaryConfig : %v.%v", err, canaryCfgMgr.logger.Error("error parsing duration - cant proceed with this canaryConfig",
canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace) zap.Error(err),
zap.String("duration", canaryConfig.Spec.WeightIncrementDuration),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return return
} }
ticker := time.NewTicker(interval) ticker := time.NewTicker(interval)
@@ -151,7 +161,11 @@ func (canaryCfgMgr *canaryConfigMgr) addCanaryConfig(canaryConfig *crd.CanaryCon
} }
err = canaryCfgMgr.canaryCfgCancelFuncMap.assign(&canaryConfig.Metadata, cacheValue) err = canaryCfgMgr.canaryCfgCancelFuncMap.assign(&canaryConfig.Metadata, cacheValue)
if err != nil { if err != nil {
log.Printf("Error caching canary config : %s.%s. err : %v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, err) canaryCfgMgr.logger.Error("error caching canary config",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return return
} }
canaryCfgMgr.processCanaryConfig(&ctx, canaryConfig, ticker) canaryCfgMgr.processCanaryConfig(&ctx, canaryConfig, ticker)
@@ -164,10 +178,17 @@ func (canaryCfgMgr *canaryConfigMgr) processCanaryConfig(ctx *context.Context, c
select { select {
case <-(*ctx).Done(): case <-(*ctx).Done():
// this case when someone deleted their canary config in the middle of it being processed // this case when someone deleted their canary config in the middle of it being processed
log.Printf("Cancel Func called for canary config : %s", canaryConfig.Metadata.Name) canaryCfgMgr.logger.Info("cancel func called for canary config",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata) err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata)
if err != nil { if err != nil {
log.Printf("error removing canary config: %s from map, err : %v", canaryConfig.Metadata.Name, err) canaryCfgMgr.logger.Error("error removing canary config",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
} }
return return
@@ -175,16 +196,26 @@ func (canaryCfgMgr *canaryConfigMgr) processCanaryConfig(ctx *context.Context, c
// every weightIncrementDuration, check if failureThreshold has reached. // every weightIncrementDuration, check if failureThreshold has reached.
// if yes, rollback. // if yes, rollback.
// else, increment the weight of new function and decrement old function by `weightIncrement` // else, increment the weight of new function and decrement old function by `weightIncrement`
log.Printf("Processing canary config : %s.%s", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace) canaryCfgMgr.logger.Info("processing canary config",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
canaryCfgMgr.RollForwardOrBack(canaryConfig, quit, ticker) canaryCfgMgr.RollForwardOrBack(canaryConfig, quit, ticker)
case <-quit: case <-quit:
// we're done processing this canary config either because the new function receives 100% of the traffic // we're done processing this canary config either because the new function receives 100% of the traffic
// or we rolled back to send all 100% traffic to old function // or we rolled back to send all 100% traffic to old function
log.Printf("Quit processing canaryConfig : %s", canaryConfig.Metadata.Name) canaryCfgMgr.logger.Info("quit processing canaryConfig",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata) err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata)
if err != nil { if err != nil {
log.Printf("error removing canary config: %s from map, err : %v", canaryConfig.Metadata.Name, err) canaryCfgMgr.logger.Error("error removing canary config from map",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
} }
return return
} }
@@ -195,7 +226,10 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
// handle race between delete event and notification on ticker.C // handle race between delete event and notification on ticker.C
_, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata) _, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
if err != nil { if err != nil {
log.Printf("No need of processing the config, not in cache anymore") canaryCfgMgr.logger.Info("no need of processing the config, not in cache anymore",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return return
} }
@@ -204,19 +238,31 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
if err != nil { if err != nil {
// if the http trigger is not found, then give up processing this config. // if the http trigger is not found, then give up processing this config.
if k8serrors.IsNotFound(err) { if k8serrors.IsNotFound(err) {
log.Printf("Http trigger object : %v.%v missing", canaryConfig.Spec.Trigger, canaryConfig.Metadata.Namespace) canaryCfgMgr.logger.Error("http trigger object for canary config missing",
zap.Error(err),
zap.String("trigger", canaryConfig.Spec.Trigger),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
close(quit) close(quit)
return return
} }
// just silently ignore. wait for next window to increment weight // just silently ignore. wait for next window to increment weight
log.Printf("Error fetching http trigger object, err : %v", err) canaryCfgMgr.logger.Error("error fetching http trigger object for config",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return return
} }
// handle a race between ticker.Stop and receiving a notification on ticker.C // handle a race between ticker.Stop and receiving a notification on ticker.C
if canaryConfig.Status.Status != fission.CanaryConfigStatusPending { if canaryConfig.Status.Status != fission.CanaryConfigStatusPending {
log.Printf("No need of processing the config, not pending anymore") canaryCfgMgr.logger.Info("no need of processing the config, not pending anymore",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return return
} }
@@ -227,22 +273,42 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
if err != nil { if err != nil {
// silently ignore. wait for next window to increment weight // silently ignore. wait for next window to increment weight
log.Printf("Error calculating failure percentage, err : %v", err) canaryCfgMgr.logger.Info("error calculating failure percentage",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return return
} }
log.Printf("Failure percentage calculated : %v for canaryConfig %s", failurePercent, canaryConfig.Metadata.Name) canaryCfgMgr.logger.Info("failure percentage calculated for canaryConfig",
zap.Float64("failure_percent", failurePercent),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
if failurePercent == -1 { if failurePercent == -1 {
// this means there were no requests triggered to this url during this window. return here and check back // this means there were no requests triggered to this url during this window. return here and check back
// during next iteration // during next iteration
log.Printf("Total requests received for url : %v is 0", triggerObj.Spec.RelativeURL) canaryCfgMgr.logger.Info("total requests received for url is 0", zap.String("url", triggerObj.Spec.RelativeURL))
return return
} }
if int(failurePercent) > canaryConfig.Spec.FailureThreshold { if int(failurePercent) > canaryConfig.Spec.FailureThreshold {
log.Printf("Failure percent %v crossed the threshold %v, so rolling back", failurePercent, canaryConfig.Spec.FailureThreshold) canaryCfgMgr.logger.Error("failure percent crossed the threshold, so rolling back",
zap.Float64("failure_percent", failurePercent),
zap.Int("threshold", canaryConfig.Spec.FailureThreshold),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
ticker.Stop() ticker.Stop()
canaryCfgMgr.rollback(canaryConfig, triggerObj) err := canaryCfgMgr.rollback(canaryConfig, triggerObj)
if err != nil {
canaryCfgMgr.logger.Error("error rolling back canary config",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
}
close(quit) close(quit)
return return
} }
@@ -251,7 +317,12 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
doneProcessingCanaryConfig, err := canaryCfgMgr.rollForward(canaryConfig, triggerObj) doneProcessingCanaryConfig, err := canaryCfgMgr.rollForward(canaryConfig, triggerObj)
if err != nil { if err != nil {
// just log the error and hope that next iteration will succeed // just log the error and hope that next iteration will succeed
log.Printf("Error incrementing weights for triggerObj : %v, err : %v", triggerObj.Metadata.Name, err) canaryCfgMgr.logger.Error("error incrementing weights for trigger",
zap.Error(err),
zap.String("trigger", triggerObj.Metadata.Name),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return return
} }
@@ -263,11 +334,17 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
fission.CanaryConfigStatusSucceeded) fission.CanaryConfigStatusSucceeded)
if err != nil { if err != nil {
// cant do much after max retries other than logging it. // cant do much after max retries other than logging it.
log.Printf("Error updating canary config : %s.%s after max retries, err :%v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, canaryCfgMgr.logger.Error("error updating canary config after max retries",
err) zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
} }
log.Printf("We're done processing canary config : %s. The new function is receiving all the traffic", canaryConfig.Metadata.Name) canaryCfgMgr.logger.Info("done processing canary config - the new function is receiving all the traffic",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
close(quit) close(quit)
return return
} }
@@ -277,8 +354,9 @@ func (canaryCfgMgr *canaryConfigMgr) updateHttpTriggerWithRetries(triggerName, t
for i := 0; i < fission.MaxRetries; i++ { for i := 0; i < fission.MaxRetries; i++ {
triggerObj, err := canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Get(triggerName) triggerObj, err := canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Get(triggerName)
if err != nil { if err != nil {
log.Printf("Error getting http trigger object : %v", err) e := "error getting http trigger object"
return err canaryCfgMgr.logger.Error(e, zap.Error(err), zap.String("trigger_name", triggerName), zap.String("trigger_namespace", triggerNamespace))
return errors.Wrap(err, e)
} }
triggerObj.Spec.FunctionReference.FunctionWeights = fnWeights triggerObj.Spec.FunctionReference.FunctionWeights = fnWeights
@@ -286,14 +364,21 @@ func (canaryCfgMgr *canaryConfigMgr) updateHttpTriggerWithRetries(triggerName, t
_, err = canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Update(triggerObj) _, err = canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Update(triggerObj)
switch { switch {
case err == nil: case err == nil:
log.Printf("Updated Http trigger : %s.%s", triggerName, triggerNamespace) canaryCfgMgr.logger.Info("updated http trigger", zap.String("trigger_name", triggerName), zap.String("trigger_namespace", triggerNamespace))
return nil return nil
case k8serrors.IsConflict(err): case k8serrors.IsConflict(err):
log.Printf("Conflict in updating http trigger : %s.%s, retrying", triggerName, triggerNamespace) canaryCfgMgr.logger.Info("conflict in updating http trigger, retrying",
zap.Error(err),
zap.String("trigger_name", triggerName),
zap.String("trigger_namespace", triggerNamespace))
continue continue
default: default:
log.Printf("Error updating trigger : %s.%s = %v", triggerName, triggerNamespace, err) e := "error updating http trigger"
return err canaryCfgMgr.logger.Info(e,
zap.Error(err),
zap.String("trigger_name", triggerName),
zap.String("trigger_namespace", triggerNamespace))
return errors.Wrapf(err, "%s: %s.%s", e, triggerName, triggerNamespace)
} }
} }
@@ -304,24 +389,42 @@ func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfigStatusWithRetries(cfgName
for i := 0; i < fission.MaxRetries; i++ { for i := 0; i < fission.MaxRetries; i++ {
canaryCfgObj, err := canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Get(cfgName) canaryCfgObj, err := canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Get(cfgName)
if err != nil { if err != nil {
log.Printf("Error getting http Canary Config object : %v", err) e := "error getting http canary config object"
return err canaryCfgMgr.logger.Error(e,
zap.Error(err),
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace),
zap.String("status", status))
return errors.Wrap(err, e)
} }
log.Printf("Updating status of canaryCfg : %s.%s to %s", cfgName, cfgNamespace, status) canaryCfgMgr.logger.Info("updating status of canary config",
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace),
zap.String("status", status))
canaryCfgObj.Status.Status = status canaryCfgObj.Status.Status = status
_, err = canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Update(canaryCfgObj) _, err = canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Update(canaryCfgObj)
switch { switch {
case err == nil: case err == nil:
log.Printf("Updated Canary Config : %s.%s", cfgName, cfgNamespace) canaryCfgMgr.logger.Info("updated canary config",
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace))
return nil return nil
case k8serrors.IsConflict(err): case k8serrors.IsConflict(err):
log.Printf("Conflict in updating Canary Config : %s.%s, retrying", cfgName, cfgNamespace) canaryCfgMgr.logger.Info("conflict in updating canary config",
zap.Error(err),
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace))
continue continue
default: default:
log.Printf("Error updating Canary Config : %s.%s = %v", cfgName, cfgNamespace, err) e := "error updating canary config"
return err canaryCfgMgr.logger.Error(e,
zap.Error(err),
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace))
return errors.Wrapf(err, "%s: %s.%s", e, cfgName, cfgNamespace)
} }
} }
@@ -358,7 +461,7 @@ func (canaryCfgMgr *canaryConfigMgr) rollForward(canaryConfig *crd.CanaryConfig,
} }
} }
log.Printf("Incremented functionWeights : %v", functionWeights) canaryCfgMgr.logger.Info("incremented functionWeights", zap.Any("function_weights", functionWeights))
err := canaryCfgMgr.updateHttpTriggerWithRetries(trigger.Metadata.Name, trigger.Metadata.Namespace, functionWeights) err := canaryCfgMgr.updateHttpTriggerWithRetries(trigger.Metadata.Name, trigger.Metadata.Namespace, functionWeights)
return doneProcessingCanaryConfig, err return doneProcessingCanaryConfig, err
@@ -369,7 +472,10 @@ func (canaryCfgMgr *canaryConfigMgr) reSyncCanaryConfigs() {
canaryConfig := obj.(*crd.CanaryConfig) canaryConfig := obj.(*crd.CanaryConfig)
_, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata) _, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
if err != nil && canaryConfig.Status.Status == fission.CanaryConfigStatusPending { if err != nil && canaryConfig.Status.Status == fission.CanaryConfigStatusPending {
log.Printf("Adding canary config : %s.%s from resync loop", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace) canaryCfgMgr.logger.Info("adding canary config from resync loop",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
// new canaryConfig detected, add it to our cache and start processing it // new canaryConfig detected, add it to our cache and start processing it
go canaryCfgMgr.addCanaryConfig(canaryConfig) go canaryCfgMgr.addCanaryConfig(canaryConfig)
@@ -378,10 +484,17 @@ func (canaryCfgMgr *canaryConfigMgr) reSyncCanaryConfigs() {
} }
func (canaryCfgMgr *canaryConfigMgr) deleteCanaryConfig(canaryConfig *crd.CanaryConfig) { func (canaryCfgMgr *canaryConfigMgr) deleteCanaryConfig(canaryConfig *crd.CanaryConfig) {
log.Printf("Delete event received for canary config : %v, %v, %v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, canaryConfig.Metadata.ResourceVersion) canaryCfgMgr.logger.Info("delete event received for canary config",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
canaryProcessingInfo, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata) canaryProcessingInfo, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
if err != nil { if err != nil {
log.Printf("lookup of canaryConfig failed, err : %v", err) canaryCfgMgr.logger.Error("lookup of canary config for deletion failed",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return return
} }
// first stop the ticker // first stop the ticker
@@ -396,7 +509,11 @@ func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfig(oldCanaryConfig *crd.Can
err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&oldCanaryConfig.Metadata) err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&oldCanaryConfig.Metadata)
if err != nil { if err != nil {
log.Printf("error removing canary config: %s from map, err : %v", oldCanaryConfig.Metadata.Name, err) canaryCfgMgr.logger.Error("error removing canary config from map",
zap.Error(err),
zap.String("name", oldCanaryConfig.Metadata.Name),
zap.String("namespace", oldCanaryConfig.Metadata.Namespace),
zap.String("version", oldCanaryConfig.Metadata.ResourceVersion))
return return
} }
canaryCfgMgr.addCanaryConfig(newCanaryConfig) canaryCfgMgr.addCanaryConfig(newCanaryConfig)
canaryconfigmgr/prometheusClient.go
+27 -34
View File
@@ -20,26 +20,26 @@ import (
"fmt" "fmt"
"time" "time"
"golang.org/x/net/context" "github.com/pkg/errors"
promClient "github.com/prometheus/client_golang/api/prometheus" promClient "github.com/prometheus/client_golang/api/prometheus"
"github.com/prometheus/common/model" "github.com/prometheus/common/model"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
"golang.org/x/net/context"
) )
type PrometheusApiClient struct { type PrometheusApiClient struct {
logger *zap.Logger
client promClient.QueryAPI client promClient.QueryAPI
} }
func MakePrometheusClient(prometheusSvc string) (*PrometheusApiClient, error) { func MakePrometheusClient(logger *zap.Logger, prometheusSvc string) (*PrometheusApiClient, error) {
promApiConfig := promClient.Config{ promApiConfig := promClient.Config{
Address: prometheusSvc, Address: prometheusSvc,
} }
promApiClient, err := promClient.New(promApiConfig) promApiClient, err := promClient.New(promApiConfig)
if err != nil { if err != nil {
log.Errorf("Error creating prometheus api client for svc : %s, err : %v", prometheusSvc, err) return nil, errors.Wrapf(err, "error creating prometheus api client for svc: %s", prometheusSvc)
return nil, err
} }
apiQueryClient := promClient.NewQueryAPI(promApiClient) apiQueryClient := promClient.NewQueryAPI(promApiClient)
@@ -56,12 +56,12 @@ func MakePrometheusClient(prometheusSvc string) (*PrometheusApiClient, error) {
} }
if err != nil { if err != nil {
log.Printf("Error sending test query to prometheus server: %v", err) return nil, errors.Wrap(err, "error sending test query to prometheus server")
return nil, err
} }
log.Printf("Successfully made prometheus client with service : %s", prometheusSvc) logger.Info("successfully made prometheus client with service", zap.String("service", prometheusSvc))
return &PrometheusApiClient{ return &PrometheusApiClient{
logger: logger.Named("prometheus_api_client"),
client: apiQueryClient, client: apiQueryClient,
}, nil }, nil
} }
@@ -74,7 +74,7 @@ func (promApiClient *PrometheusApiClient) GetFunctionFailurePercentage(path, met
} }
if reqs <= 0 { if reqs <= 0 {
return -1, fmt.Errorf("no requests to this url %v and method %v in the window : %v", path, method, window) return -1, fmt.Errorf("no requests to this url %v and method %v in the window: %v", path, method, window)
} }
// next, get a total count of errored out requests to this function in the same window // next, get a total count of errored out requests to this function in the same window
@@ -94,20 +94,22 @@ func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string,
reqs, err := promApiClient.executeQuery(queryString) reqs, err := promApiClient.executeQuery(queryString)
if err != nil { if err != nil {
log.Printf("Error executing query : %s, err : %v", queryString, err) return 0, errors.Wrapf(err, "error executing query: %s", queryString)
return 0, err
} }
queryString = fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"} offset %v", path, method, funcName, funcNs, window) queryString = fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"} offset %v", path, method, funcName, funcNs, window)
reqsInPrevWindow, err := promApiClient.executeQuery(queryString) reqsInPrevWindow, err := promApiClient.executeQuery(queryString)
if err != nil { if err != nil {
log.Printf("Error executing query : %s, err : %v", queryString, err) return 0, errors.Wrapf(err, "error executing query: %s", queryString)
return 0, err
} }
reqsInCurrentWindow := reqs - reqsInPrevWindow reqsInCurrentWindow := reqs - reqsInPrevWindow
log.Printf("reqs : %v, reqsInPrevWindow : %v, reqsInCurrentWindow : %v to function %v", reqs, reqsInPrevWindow, reqsInCurrentWindow, funcName) promApiClient.logger.Info("function requests",
zap.Float64("requests", reqs),
zap.Float64("requests_in_previous_window", reqsInPrevWindow),
zap.Float64("requests_in_current_window", reqsInCurrentWindow),
zap.String("function", funcName))
return reqsInCurrentWindow, nil return reqsInCurrentWindow, nil
} }
@@ -117,20 +119,22 @@ func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(f
failedRequests, err := promApiClient.executeQuery(queryString) failedRequests, err := promApiClient.executeQuery(queryString)
if err != nil { if err != nil {
log.Printf("Error executing query : %s, err : %v", queryString, err) return 0, errors.Wrapf(err, "error executing query: %s", queryString)
return 0, err
} }
queryString = fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"} offset %v", funcName, funcNs, path, method, window) queryString = fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"} offset %v", funcName, funcNs, path, method, window)
failedReqsInPrevWindow, err := promApiClient.executeQuery(queryString) failedReqsInPrevWindow, err := promApiClient.executeQuery(queryString)
if err != nil { if err != nil {
log.Printf("Error executing query : %s, err : %v", queryString, err) return 0, errors.Wrapf(err, "error executing query: %s", queryString)
return 0, err
} }
failedReqsInCurrentWindow := failedRequests - failedReqsInPrevWindow failedReqsInCurrentWindow := failedRequests - failedReqsInPrevWindow
log.Printf("failedReqs : %v, failedReqsInPrevWindow : %v, failedReqsInCurrentWindow : %v to function : %v", failedRequests, failedReqsInPrevWindow, failedReqsInCurrentWindow, funcName) promApiClient.logger.Info("function requests",
zap.Float64("failed_requests", failedRequests),
zap.Float64("failed_requests_in_previous_window", failedReqsInPrevWindow),
zap.Float64("failed_requests_in_current_window", failedReqsInCurrentWindow),
zap.String("function", funcName))
return failedReqsInCurrentWindow, nil return failedReqsInCurrentWindow, nil
} }
@@ -138,8 +142,7 @@ func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(f
func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (float64, error) { func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (float64, error) {
val, err := promApiClient.client.Query(context.Background(), queryString, time.Now()) val, err := promApiClient.client.Query(context.Background(), queryString, time.Now())
if err != nil { if err != nil {
log.Errorf("Error querying prometheus qs : %v, err : %v", queryString, err) return 0, errors.Wrapf(err, "error querying prometheus")
return 0, err
} }
switch { switch {
@@ -159,23 +162,13 @@ func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (floa
matrixVal := val.(model.Matrix) matrixVal := val.(model.Matrix)
total := float64(0) total := float64(0)
for _, elem := range matrixVal { for _, elem := range matrixVal {
//log.Printf("Only one value, so taking the 0th elem")
total += float64(elem.Values[len(elem.Values)-1].Value) total += float64(elem.Values[len(elem.Values)-1].Value)
} }
return total, nil return total, nil
default: default:
log.Printf("type unrecognized") promApiClient.logger.Info("return value type of prometheus query was unrecognized",
zap.Any("type", val.Type()))
return 0, nil return 0, nil
} }
} }
func addInterval(window string) string {
timeDuration, _ := time.ParseDuration(window)
log.Println("window in seconds", int64(timeDuration/time.Second))
timeInStr := fmt.Sprintf("%ds", int64((timeDuration+timeDuration)/time.Second))
fmt.Println(timeInStr)
return timeInStr
}
charts/fission-all/templates/fluentd.yaml
+1 -1
View File
@@ -28,7 +28,7 @@ spec:
spec: spec:
containers: containers:
- name: fluentd - name: fluentd
image: "{{ .Values.repository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}" image: "{{ .Values.logger.fluentdImageRepository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}"
imagePullPolicy: {{ .Values.pullPolicy }} imagePullPolicy: {{ .Values.pullPolicy }}
env: env:
- name: INFLUXDB_ADDRESS - name: INFLUXDB_ADDRESS
charts/fission-all/values.yaml
+1
View File
@@ -53,6 +53,7 @@ enableIstio: false
## Logger config ## Logger config
logger: logger:
influxdbAdmin: "admin" influxdbAdmin: "admin"
fluentdImageRepository: index.docker.io
fluentdImage: fission/fluentd fluentdImage: fission/fluentd
fluentdImageTag: 1.0.0 fluentdImageTag: 1.0.0
common.go
+28 -8
View File
@@ -18,6 +18,7 @@ package fission
import ( import (
"fmt" "fmt"
"io"
"io/ioutil" "io/ioutil"
"net" "net"
"net/http" "net/http"
@@ -28,6 +29,8 @@ import (
"strings" "strings"
"syscall" "syscall"
"go.uber.org/zap"
"github.com/gorilla/handlers" "github.com/gorilla/handlers"
"github.com/imdario/mergo" "github.com/imdario/mergo"
"github.com/mholt/archiver" "github.com/mholt/archiver"
@@ -67,14 +70,31 @@ func GetFunctionIstioServiceName(fnName, fnNamespace string) string {
return fmt.Sprintf("istio-%v-%v", fnName, fnNamespace) return fmt.Sprintf("istio-%v-%v", fnName, fnNamespace)
} }
func LoggingMiddleware(next http.Handler) http.Handler { func LoggingMiddleware(logger *zap.Logger) func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return func(next http.Handler) http.Handler {
requestURI := r.RequestURI return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.Contains(requestURI, "healthz") { requestURI := r.RequestURI
// Call the next handler, which can be another middleware in the chain, or the final handler. if !strings.Contains(requestURI, "healthz") {
handlers.LoggingHandler(os.Stdout, next).ServeHTTP(w, r) // Call the next handler, which can be another middleware in the chain, or the final handler.
} handlers.CustomLoggingHandler(os.Stdout, next, func(writer io.Writer, params handlers.LogFormatterParams) {
}) host, _, err := net.SplitHostPort(params.Request.RemoteAddr)
if err != nil {
host = params.Request.RemoteAddr
}
logger.Info("handled",
zap.String("host", host),
zap.String("method", params.Request.Method),
zap.String("uri", params.Request.RequestURI),
zap.String("proto", params.Request.Proto),
zap.Int("status_code", params.StatusCode),
zap.Int("size", params.Size))
}).ServeHTTP(w, r)
}
})
}
} }
// MergeContainerSpecs merges container specs using a predefined order. // MergeContainerSpecs merges container specs using a predefined order.
commonrbacutil.go
+62 -26
View File
@@ -18,8 +18,10 @@ package fission
import ( import (
"fmt" "fmt"
"log"
"go.uber.org/zap"
"github.com/pkg/errors"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1beta1" rbac "k8s.io/api/rbac/v1beta1"
k8serrors "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -101,7 +103,7 @@ type PatchSpec struct {
} }
// AddSaToRoleBindingWithRetries adds a service account to a rolebinding object. IT retries on already exists and conflict errors. // AddSaToRoleBindingWithRetries adds a service account to a rolebinding object. IT retries on already exists and conflict errors.
func AddSaToRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind string) (err error) { func AddSaToRoleBindingWithRetries(logger *zap.Logger, k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind string) (err error) {
patch := PatchSpec{} patch := PatchSpec{}
patch.Op = "add" patch.Op = "add"
patch.Path = "/subjects/-" patch.Path = "/subjects/-"
@@ -113,34 +115,44 @@ func AddSaToRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding,
patchJson, err := json.Marshal([]PatchSpec{patch}) patchJson, err := json.Marshal([]PatchSpec{patch})
if err != nil { if err != nil {
log.Printf("Error marshalling patch into json") logger.Error("error marshalling patch into json", zap.Error(err))
return err return err
} }
for i := 0; i < MaxRetries; i++ { for i := 0; i < MaxRetries; i++ {
_, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Patch(roleBinding, types.JSONPatchType, patchJson) _, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Patch(roleBinding, types.JSONPatchType, patchJson)
if err == nil { if err == nil {
log.Printf("Patched rolebinding : %s.%s", roleBinding, roleBindingNs) logger.Info("patched rolebinding",
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return err return err
} }
if k8serrors.IsNotFound(err) { if k8serrors.IsNotFound(err) {
logger.Info("rolebinding not found - will try to create it",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
// someone may have deleted the object between us checking if the object is present and deciding to patch // someone may have deleted the object between us checking if the object is present and deciding to patch
// so just create the object again // so just create the object again
rbObj := makeRoleBindingObj(roleBinding, roleBindingNs, role, roleKind, sa, saNamespace) rbObj := makeRoleBindingObj(roleBinding, roleBindingNs, role, roleKind, sa, saNamespace)
rbObj, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Create(rbObj) rbObj, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Create(rbObj)
if err == nil { if err == nil {
log.Printf("Created rolebinding : %s.%s", roleBinding, roleBindingNs) logger.Info("created rolebinding",
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return err return err
} }
if k8serrors.IsAlreadyExists(err) { if k8serrors.IsAlreadyExists(err) {
log.Printf("rolebinding object : %s.%s already exists, retrying patch, err: %v", roleBinding, roleBindingNs, err) logger.Info("rolebinding object already exists, retrying patch",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
continue continue
} }
log.Printf("Error returned by rolebinding create, %v", err) return errors.Wrap(err, "error returned by rolebinding create")
return err
} }
if k8serrors.IsConflict(err) { if k8serrors.IsConflict(err) {
@@ -149,27 +161,30 @@ func AddSaToRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding,
// but one CI run did show patch errored out on conflict : https://api.travis-ci.org/v3/job/373161490/log.txt, look for : // but one CI run did show patch errored out on conflict : https://api.travis-ci.org/v3/job/373161490/log.txt, look for :
// Error returned by rolebinding patch : <some more text> there is a meaningful conflict (firstResourceVersion: "35482724", currentResourceVersion: "35482849") // Error returned by rolebinding patch : <some more text> there is a meaningful conflict (firstResourceVersion: "35482724", currentResourceVersion: "35482849")
// so, m guessing retrying patch should help. will watch out for any such conflicts and fix the issue if any // so, m guessing retrying patch should help. will watch out for any such conflicts and fix the issue if any
log.Printf("Conflict reported on patch of rolebinding : %s.%s. Retrying patch operation, err : %v", roleBinding, roleBindingNs, err) logger.Info("conflict reported on patch of rolebinding - retrying patch operation",
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
continue continue
} }
log.Printf("Error returned by rolebinding patch : %v", err) return errors.Wrap(err, "error returned by rolebinding patch")
return err
} }
log.Printf("Exceeded maxretries : %d adding SA: %s.%s to rolebinding : %s.%s, giving up, err: %v", MaxRetries, sa, saNamespace, roleBinding, roleBindingNs, err) return errors.Wrapf(err, "exceeded max retries (%d) adding SA: %s.%s to rolebinding: %s.%s, giving up", MaxRetries, sa, saNamespace, roleBinding, roleBindingNs)
return err
} }
// RemoveSAFromRoleBindingWithRetries removes an SA from the rolebinding passed as parameter. If this is the only SA in // RemoveSAFromRoleBindingWithRetries removes an SA from the rolebinding passed as parameter. If this is the only SA in
// the rolebinding, then it deletes the rolebinding object. // the rolebinding, then it deletes the rolebinding object.
func RemoveSAFromRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs string, saToRemove map[string]bool) (err error) { func RemoveSAFromRoleBindingWithRetries(logger *zap.Logger, k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs string, saToRemove map[string]bool) (err error) {
for i := 0; i < MaxRetries; i++ { for i := 0; i < MaxRetries; i++ {
rbObj, err := k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Get( rbObj, err := k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Get(
roleBinding, metav1.GetOptions{}) roleBinding, metav1.GetOptions{})
if err != nil { if err != nil {
// silently ignoring the error. there's no need for us to remove sa anymore. // silently ignoring the error. there's no need for us to remove sa anymore.
log.Printf("rolebinding %s.%s not found, but ignoring the error since we're cleaning up", roleBinding, roleBindingNs) logger.Info("rolebinding not found, but ignoring the error since we're cleaning up",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return nil return nil
} }
@@ -198,43 +213,64 @@ func RemoveSAFromRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBin
_, err = k8sClient.RbacV1beta1().RoleBindings(rbObj.Namespace).Update(rbObj) _, err = k8sClient.RbacV1beta1().RoleBindings(rbObj.Namespace).Update(rbObj)
switch { switch {
case err == nil: case err == nil:
log.Printf("Removed sa's : %v from rolebinding : %s.%s", saToRemove, roleBinding, roleBindingNs) logger.Info("removed service accounts from rolebinding",
zap.Any("service_accounts", saToRemove),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return nil return nil
case k8serrors.IsConflict(err): case k8serrors.IsConflict(err):
log.Printf("Conflict in update of rolebinding: %s.%s, retrying", roleBinding, roleBindingNs) logger.Info("conflict in update of rolebinding - retrying",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
continue continue
default: default:
log.Printf("Rolebinding Update Errored out : %v", err) return errors.Wrap(err, "rolebinding update errored out")
return err
} }
} }
log.Printf("Maxretries : %d exceeded for removing SA's : %v from rolebinding %s.%s, giving up, err: %v", MaxRetries, saToRemove, roleBinding, roleBindingNs, err) return errors.Wrapf(err, "max retries: %d exceeded for removing SA's: %v from rolebinding %s.%s, giving up", MaxRetries, saToRemove, roleBinding, roleBindingNs)
return err
} }
// SetupRoleBinding adds a role to a service account if the rolebinding object is already present in the namespace. // SetupRoleBinding adds a role to a service account if the rolebinding object is already present in the namespace.
// if not, it creates a rolebinding object granting the role to the SA in the namespace. // if not, it creates a rolebinding object granting the role to the SA in the namespace.
func SetupRoleBinding(k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, role, roleKind, sa, saNamespace string) error { func SetupRoleBinding(logger *zap.Logger, k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, role, roleKind, sa, saNamespace string) error {
// get the role binding object // get the role binding object
rbObj, err := k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Get( rbObj, err := k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Get(
roleBinding, metav1.GetOptions{}) roleBinding, metav1.GetOptions{})
if err == nil { if err == nil {
if !isSAInRoleBinding(rbObj, sa, saNamespace) { if !isSAInRoleBinding(rbObj, sa, saNamespace) {
return AddSaToRoleBindingWithRetries(k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind) logger.Info("service account is not present in the rolebinding - will add",
zap.String("service_account_name", sa),
zap.String("service_account_namespace", saNamespace),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return AddSaToRoleBindingWithRetries(logger, k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind)
} }
log.Printf("SA : %s.%s already present in rolebinding : %s.%s, so nothing to add", sa, saNamespace, roleBinding, roleBindingNs) logger.Info("service account already present in rolebinding so nothing to add",
zap.String("service_account_name", sa),
zap.String("service_account_namespace", saNamespace),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return nil return nil
} }
// if role binding is missing, create it. also add this sa to the binding. // if role binding is missing, create it. also add this sa to the binding.
if k8serrors.IsNotFound(err) { if k8serrors.IsNotFound(err) {
log.Printf("Rolebinding %s does NOT exist in ns %s. Creating it", roleBinding, roleBindingNs) logger.Info("rolebinding does NOT exist in namespace - creating it",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
rbObj = makeRoleBindingObj(roleBinding, roleBindingNs, role, roleKind, sa, saNamespace) rbObj = makeRoleBindingObj(roleBinding, roleBindingNs, role, roleKind, sa, saNamespace)
rbObj, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Create(rbObj) rbObj, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Create(rbObj)
if k8serrors.IsAlreadyExists(err) { if k8serrors.IsAlreadyExists(err) {
err = AddSaToRoleBindingWithRetries(k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind) logger.Info("rolebinding already exists in namespace - adding service account to rolebinding",
zap.String("service_account_name", sa),
zap.String("service_account_namespace", saNamespace),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
err = AddSaToRoleBindingWithRetries(logger, k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind)
} }
} }
controller/api.go
+9 -7
View File
@@ -24,7 +24,7 @@ import (
"strings" "strings"
"github.com/gorilla/mux" "github.com/gorilla/mux"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors" kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,6 +46,7 @@ func init() {
type ( type (
API struct { API struct {
logger *zap.Logger
fissionClient *crd.FissionClient fissionClient *crd.FissionClient
kubernetesClient *kubernetes.Clientset kubernetesClient *kubernetes.Clientset
storageServiceUrl string storageServiceUrl string
@@ -63,8 +64,8 @@ type (
} }
) )
func MakeAPI(featureStatus map[string]string) (*API, error) { func MakeAPI(logger *zap.Logger, featureStatus map[string]string) (*API, error) {
api, err := makeCRDBackedAPI() api, err := makeCRDBackedAPI(logger)
u := os.Getenv("STORAGE_SERVICE_URL") u := os.Getenv("STORAGE_SERVICE_URL")
if len(u) > 0 { if len(u) > 0 {
@@ -119,7 +120,7 @@ func (api *API) respondWithError(w http.ResponseWriter, err error) {
} }
code, msg := fission.GetHTTPError(err) code, msg := fission.GetHTTPError(err)
log.Errorf("Error: %v: %v", code, msg) api.logger.Error(msg, zap.Int("code", code))
http.Error(w, msg, code) http.Error(w, msg, code)
} }
@@ -270,7 +271,8 @@ func (api *API) Serve(port int) {
address := fmt.Sprintf(":%v", port) address := fmt.Sprintf(":%v", port)
log.WithFields(log.Fields{"port": port}).Info("Server started") api.logger.Info("server started", zap.Int("port", port))
r.Use(fission.LoggingMiddleware) r.Use(fission.LoggingMiddleware(api.logger))
log.Fatal(http.ListenAndServe(address, r)) err := http.ListenAndServe(address, r)
api.logger.Fatal("done listening", zap.Error(err))
} }
controller/api_test.go
+5 -1
View File
@@ -28,6 +28,7 @@ import (
"testing" "testing"
"time" "time"
"go.uber.org/zap"
"k8s.io/api/core/v1" "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -344,7 +345,10 @@ func TestMain(m *testing.M) {
return return
} }
go Start(8888, true) logger, err := zap.NewDevelopment()
panicIf(err)
go Start(logger, 8888, true)
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
g.client = client.MakeClient("http://localhost:8888") g.client = client.MakeClient("http://localhost:8888")
controller/canaryConfigApi.go
+2 -2
View File
@@ -23,7 +23,7 @@ import (
"net/http" "net/http"
"github.com/gorilla/mux" "github.com/gorilla/mux"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
@@ -47,7 +47,7 @@ func (a *API) CanaryConfigApiCreate(w http.ResponseWriter, r *http.Request) {
var canaryCfg crd.CanaryConfig var canaryCfg crd.CanaryConfig
err = json.Unmarshal(body, &canaryCfg) err = json.Unmarshal(body, &canaryCfg)
if err != nil { if err != nil {
log.Printf("Failed to unmarshal request body: [%v]", body) a.logger.Error("failed to unmarshal request body", zap.Error(err), zap.Binary("body", body))
a.respondWithError(w, err) a.respondWithError(w, err)
return return
} }
controller/config.go
+9 -9
View File
@@ -18,9 +18,9 @@ package controller
import ( import (
"context" "context"
"fmt"
log "github.com/sirupsen/logrus" "github.com/pkg/errors"
"go.uber.org/zap"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
"github.com/fission/fission/canaryconfigmgr" "github.com/fission/fission/canaryconfigmgr"
@@ -28,24 +28,24 @@ import (
config "github.com/fission/fission/featureconfig" config "github.com/fission/fission/featureconfig"
) )
func ConfigCanaryFeature(context context.Context, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, featureConfig *config.FeatureConfig, featureStatus map[string]string) error { func ConfigCanaryFeature(context context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, featureConfig *config.FeatureConfig, featureStatus map[string]string) error {
// start the appropriate controller // start the appropriate controller
if featureConfig.CanaryConfig.IsEnabled { if featureConfig.CanaryConfig.IsEnabled {
canaryCfgMgr, err := canaryconfigmgr.MakeCanaryConfigMgr(fissionClient, kubeClient, fissionClient.GetCrdClient(), canaryCfgMgr, err := canaryconfigmgr.MakeCanaryConfigMgr(logger, fissionClient, kubeClient, fissionClient.GetCrdClient(),
featureConfig.CanaryConfig.PrometheusSvc) featureConfig.CanaryConfig.PrometheusSvc)
if err != nil { if err != nil {
featureStatus[config.CanaryFeature] = err.Error() featureStatus[config.CanaryFeature] = err.Error()
return fmt.Errorf("failed to start canary config manager: %v", err) return errors.Wrap(err, "failed to start canary config manager")
} }
canaryCfgMgr.Run(context) canaryCfgMgr.Run(context)
log.Printf("Started canary config manager") logger.Info("started canary config manager")
} }
return nil return nil
} }
// ConfigureFeatures gets the feature config and configures the features that are enabled // ConfigureFeatures gets the feature config and configures the features that are enabled
func ConfigureFeatures(context context.Context, unitTestMode bool, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset) (map[string]string, error) { func ConfigureFeatures(context context.Context, logger *zap.Logger, unitTestMode bool, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset) (map[string]string, error) {
// set feature enabled to false if unitTestMode // set feature enabled to false if unitTestMode
if unitTestMode { if unitTestMode {
return nil, nil return nil, nil
@@ -54,7 +54,7 @@ func ConfigureFeatures(context context.Context, unitTestMode bool, fissionClient
// get the featureConfig from config map mounted onto the file system // get the featureConfig from config map mounted onto the file system
featureConfig, err := config.GetFeatureConfig() featureConfig, err := config.GetFeatureConfig()
if err != nil { if err != nil {
log.Printf("Error getting feature config : %v", err) logger.Error("error getting feature config", zap.Error(err))
return nil, err return nil, err
} }
@@ -62,6 +62,6 @@ func ConfigureFeatures(context context.Context, unitTestMode bool, fissionClient
// configure respective features // configure respective features
// in the future when new optional features are added, we need to add corresponding feature handlers and invoke them here // in the future when new optional features are added, we need to add corresponding feature handlers and invoke them here
err = ConfigCanaryFeature(context, fissionClient, kubeClient, featureConfig, featureStatus) err = ConfigCanaryFeature(context, logger, fissionClient, kubeClient, featureConfig, featureStatus)
return featureStatus, err return featureStatus, err
} }
controller/configmapApi.go
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"net/http" "net/http"
"github.com/gorilla/mux" "github.com/gorilla/mux"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
@@ -35,7 +35,7 @@ func (a *API) ConfigMapGet(w http.ResponseWriter, r *http.Request) {
configMap, err := a.kubernetesClient.CoreV1().ConfigMaps(ns).Get(name, metav1.GetOptions{}) configMap, err := a.kubernetesClient.CoreV1().ConfigMaps(ns).Get(name, metav1.GetOptions{})
if err != nil { if err != nil {
log.Printf("Error getting config map: %s from ns: %s", name, ns) a.logger.Error("error getting config map", zap.Error(err), zap.String("config_map_name", name), zap.String("namespace", ns))
a.respondWithError(w, err) a.respondWithError(w, err)
return return
} }
controller/controller.go
+12 -10
View File
@@ -18,41 +18,43 @@ package controller
import ( import (
"context" "context"
"log"
"go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
) )
func Start(port int, unitTestFlag bool) { func Start(logger *zap.Logger, port int, unitTestFlag bool) {
cLogger := logger.Named("controller")
// setup a signal handler for SIGTERM // setup a signal handler for SIGTERM
fission.SetupStackTraceHandler() fission.SetupStackTraceHandler()
fc, kc, apiExtClient, err := crd.MakeFissionClient() fc, kc, apiExtClient, err := crd.MakeFissionClient()
if err != nil { if err != nil {
log.Fatalf("Failed to connect to K8s API: %v", err) cLogger.Fatal("failed to connect to k8s API", zap.Error(err))
} }
err = crd.EnsureFissionCRDs(apiExtClient) err = crd.EnsureFissionCRDs(cLogger, apiExtClient)
if err != nil { if err != nil {
log.Fatalf("Failed to create fission CRDs: %v", err) cLogger.Fatal("failed to create fission CRDs", zap.Error(err))
} }
err = fc.WaitForCRDs() err = fc.WaitForCRDs()
if err != nil { if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err) cLogger.Fatal("error waiting for CRDs", zap.Error(err))
} }
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
featureStatus, err := ConfigureFeatures(ctx, unitTestFlag, fc, kc) featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc)
if err != nil { if err != nil {
log.Printf("Error configuring features : %v. Proceeding without optional features", err.Error()) cLogger.Info("error configuring features - proceeding without optional features", zap.Error(err))
} }
defer cancel() defer cancel()
api, err := MakeAPI(featureStatus) api, err := MakeAPI(cLogger, featureStatus)
if err != nil { if err != nil {
log.Fatalf("Failed to start controller: %v", err) cLogger.Fatal("failed to start controller", zap.Error(err))
} }
api.Serve(port) api.Serve(port)
} }
controller/crd.go
+8 -2
View File
@@ -17,13 +17,19 @@ limitations under the License.
package controller package controller
import ( import (
"go.uber.org/zap"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
) )
func makeCRDBackedAPI() (*API, error) { func makeCRDBackedAPI(logger *zap.Logger) (*API, error) {
fissionClient, kubernetesClient, _, err := crd.MakeFissionClient() fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &API{fissionClient: fissionClient, kubernetesClient: kubernetesClient}, nil return &API{
logger: logger.Named("api"),
fissionClient: fissionClient,
kubernetesClient: kubernetesClient,
}, nil
} }
controller/environmentApi.go
+2 -2
View File
@@ -22,7 +22,7 @@ import (
"net/http" "net/http"
"github.com/gorilla/mux" "github.com/gorilla/mux"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
@@ -60,7 +60,7 @@ func (a *API) EnvironmentApiCreate(w http.ResponseWriter, r *http.Request) {
var env crd.Environment var env crd.Environment
err = json.Unmarshal(body, &env) err = json.Unmarshal(body, &env)
if err != nil { if err != nil {
log.Printf("Failed to unmarshal request body: [%v]", body) a.logger.Error("failed to unmarshal request body", zap.Error(err), zap.Binary("body", body))
a.respondWithError(w, err) a.respondWithError(w, err)
return return
} }
controller/functionApi.go
+4 -2
View File
@@ -27,7 +27,7 @@ import (
"sort" "sort"
"github.com/gorilla/mux" "github.com/gorilla/mux"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
restclient "k8s.io/client-go/rest" restclient "k8s.io/client-go/rest"
@@ -188,7 +188,9 @@ func (a *API) FunctionLogsApiPost(w http.ResponseWriter, r *http.Request) {
svcUrl, err := url.Parse(dbCnf.httpURL) svcUrl, err := url.Parse(dbCnf.httpURL)
if err != nil { if err != nil {
log.Printf("Failed to establish proxy server for function logs: %v", err) a.logger.Error("failed parse url to establish proxy to database for function logs",
zap.Error(err),
zap.String("database_url", dbCnf.httpURL))
} }
// set up proxy server director // set up proxy server director
director := func(req *http.Request) { director := func(req *http.Request) {
controller/recordsApi.go
+4 -4
View File
@@ -26,7 +26,7 @@ import (
) )
func (a *API) RecordsApiListAll(w http.ResponseWriter, r *http.Request) { func (a *API) RecordsApiListAll(w http.ResponseWriter, r *http.Request) {
resp, err := redis.RecordsListAll() resp, err := redis.RecordsListAll(a.logger.Named("redis"))
if err != nil { if err != nil {
a.respondWithError(w, err) a.respondWithError(w, err)
return return
@@ -50,7 +50,7 @@ func (a *API) RecordsApiFilterByFunction(w http.ResponseWriter, r *http.Request)
return return
} }
resp, err := redis.RecordsFilterByFunction(query, recorders, triggers) resp, err := redis.RecordsFilterByFunction(a.logger.Named("redis"), query, recorders, triggers)
if err != nil { if err != nil {
a.respondWithError(w, err) a.respondWithError(w, err)
return return
@@ -74,7 +74,7 @@ func (a *API) RecordsApiFilterByTrigger(w http.ResponseWriter, r *http.Request)
return return
} }
resp, err := redis.RecordsFilterByTrigger(query, recorders, triggers) resp, err := redis.RecordsFilterByTrigger(a.logger.Named("redis"), query, recorders, triggers)
if err != nil { if err != nil {
a.respondWithError(w, err) a.respondWithError(w, err)
return return
@@ -86,7 +86,7 @@ func (a *API) RecordsApiFilterByTime(w http.ResponseWriter, r *http.Request) {
from := r.FormValue("from") from := r.FormValue("from")
to := r.FormValue("to") to := r.FormValue("to")
resp, err := redis.RecordsFilterByTime(from, to) resp, err := redis.RecordsFilterByTime(a.logger.Named("redis"), from, to)
if err != nil { if err != nil {
a.respondWithError(w, err) a.respondWithError(w, err)
return return
controller/replayAPI.go
+1 -1
View File
@@ -31,7 +31,7 @@ func (a *API) ReplayByReqUID(w http.ResponseWriter, r *http.Request) {
routerUrl := fmt.Sprintf("http://router.%v", podNamespace) routerUrl := fmt.Sprintf("http://router.%v", podNamespace)
resp, err := redis.ReplayByReqUID(routerUrl, queriedID) resp, err := redis.ReplayByReqUID(a.logger, routerUrl, queriedID)
if err != nil { if err != nil {
a.respondWithError(w, err) a.respondWithError(w, err)
return return
controller/secretApi.go
+5 -2
View File
@@ -21,7 +21,7 @@ import (
"net/http" "net/http"
"github.com/gorilla/mux" "github.com/gorilla/mux"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
@@ -35,7 +35,10 @@ func (a *API) SecretGet(w http.ResponseWriter, r *http.Request) {
secret, err := a.kubernetesClient.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{}) secret, err := a.kubernetesClient.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{})
if err != nil { if err != nil {
log.Printf("Error getting secret: %s from ns: %s", name, ns) a.logger.Error("error getting secret",
zap.Error(err),
zap.String("secret_name", name),
zap.String("namespace", ns))
a.respondWithError(w, err) a.respondWithError(w, err)
return return
} }
controller/storagesvc.go
+5 -4
View File
@@ -18,19 +18,20 @@ package controller
import ( import (
"fmt" "fmt"
"log"
"net/http" "net/http"
"net/http/httputil" "net/http/httputil"
"net/url" "net/url"
"go.uber.org/zap"
) )
func (api *API) StorageServiceProxy(w http.ResponseWriter, r *http.Request) { func (api *API) StorageServiceProxy(w http.ResponseWriter, r *http.Request) {
u := api.storageServiceUrl u := api.storageServiceUrl
ssUrl, err := url.Parse(u) ssUrl, err := url.Parse(u)
if err != nil { if err != nil {
msg := fmt.Sprintf("Error parsing url %v: %v", u, err) e := "error parsing url"
log.Println(msg) api.logger.Error(e, zap.Error(err), zap.String("url", u))
http.Error(w, msg, http.StatusInternalServerError) http.Error(w, fmt.Sprintf("%s %s: %v", e, u, err), http.StatusInternalServerError)
return return
} }
director := func(req *http.Request) { director := func(req *http.Request) {
controller/workflowApiProxy.go
+2 -1
View File
@@ -2,10 +2,11 @@ package controller
import ( import (
"fmt" "fmt"
"github.com/gorilla/mux"
"net/http" "net/http"
"net/http/httputil" "net/http/httputil"
"net/url" "net/url"
"github.com/gorilla/mux"
) )
func (api *API) WorkflowApiserverProxy(w http.ResponseWriter, r *http.Request) { func (api *API) WorkflowApiserverProxy(w http.ResponseWriter, r *http.Request) {
crd/crd.go
+7 -7
View File
@@ -17,12 +17,12 @@ limitations under the License.
package crd package crd
import ( import (
"log"
"time" "time"
"go.uber.org/zap"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
@@ -34,7 +34,7 @@ const (
// ensureCRD checks if the given CRD type exists, and creates it if // ensureCRD checks if the given CRD type exists, and creates it if
// needed. (Note that this creates the CRD type; it doesn't create any // needed. (Note that this creates the CRD type; it doesn't create any
// _instances_ of that type.) // _instances_ of that type.)
func ensureCRD(clientset *apiextensionsclient.Clientset, crd *apiextensionsv1beta1.CustomResourceDefinition) (err error) { func ensureCRD(logger *zap.Logger, clientset *apiextensionsclient.Clientset, crd *apiextensionsv1beta1.CustomResourceDefinition) (err error) {
maxRetries := 5 maxRetries := 5
for i := 0; i < maxRetries; i++ { for i := 0; i < maxRetries; i++ {
@@ -44,12 +44,12 @@ func ensureCRD(clientset *apiextensionsclient.Clientset, crd *apiextensionsv1bet
} }
// return if the resource already exists // return if the resource already exists
if errors.IsAlreadyExists(err) { if k8serrors.IsAlreadyExists(err) {
return nil return nil
} else { } else {
// The requests fail to connect to k8s api server before // The requests fail to connect to k8s api server before
// istio-prxoy is ready to serve traffic. Retry again. // istio-prxoy is ready to serve traffic. Retry again.
log.Printf("Error connecting to kubernetes api service (%v), retrying", err) logger.Info("error connecting to kubernetes api service, retrying", zap.Error(err))
time.Sleep(500 * time.Duration(2*i) * time.Millisecond) time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
continue continue
} }
@@ -59,7 +59,7 @@ func ensureCRD(clientset *apiextensionsclient.Clientset, crd *apiextensionsv1bet
} }
// Ensure CRDs // Ensure CRDs
func EnsureFissionCRDs(clientset *apiextensionsclient.Clientset) error { func EnsureFissionCRDs(logger *zap.Logger, clientset *apiextensionsclient.Clientset) error {
crds := []apiextensionsv1beta1.CustomResourceDefinition{ crds := []apiextensionsv1beta1.CustomResourceDefinition{
// Functions // Functions
{ {
@@ -207,7 +207,7 @@ func EnsureFissionCRDs(clientset *apiextensionsclient.Clientset) error {
}, },
} }
for _, crd := range crds { for _, crd := range crds {
err := ensureCRD(clientset, &crd) err := ensureCRD(logger, clientset, &crd)
if err != nil { if err != nil {
return err return err
} }
crd/crd_test.go
+5 -1
View File
@@ -22,6 +22,7 @@ import (
"testing" "testing"
"time" "time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest" "k8s.io/client-go/rest"
@@ -427,8 +428,11 @@ func TestCrd(t *testing.T) {
config, _, apiExtClient, err := GetKubernetesClient() config, _, apiExtClient, err := GetKubernetesClient()
panicIf(err) panicIf(err)
logger, err := zap.NewDevelopment()
panicIf(err)
// init our types // init our types
err = EnsureFissionCRDs(apiExtClient) err = EnsureFissionCRDs(logger, apiExtClient)
panicIf(err) panicIf(err)
// rest client with knowledge about our crd types // rest client with knowledge about our crd types
crd/recorder.go
-1
View File
@@ -19,7 +19,6 @@ package crd
import ( import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch" "k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest" "k8s.io/client-go/rest"
) )
environments/fetcher/client/client.go
+13 -9
View File
@@ -5,7 +5,9 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"io/ioutil" "io/ioutil"
"log"
"go.uber.org/zap"
"net/http" "net/http"
"strings" "strings"
"time" "time"
@@ -18,14 +20,16 @@ import (
type ( type (
Client struct { Client struct {
logger *zap.Logger
url string url string
httpClient *http.Client httpClient *http.Client
} }
) )
func MakeClient(fetcherUrl string) *Client { func MakeClient(logger *zap.Logger, fetcherUrl string) *Client {
return &Client{ return &Client{
url: strings.TrimSuffix(fetcherUrl, "/"), logger: logger.Named("fetcher_client"),
url: strings.TrimSuffix(fetcherUrl, "/"),
httpClient: &http.Client{ httpClient: &http.Client{
Transport: &ochttp.Transport{}, Transport: &ochttp.Transport{},
}, },
@@ -45,17 +49,17 @@ func (c *Client) getUploadUrl() string {
} }
func (c *Client) Specialize(ctx context.Context, req *fission.FunctionSpecializeRequest) error { func (c *Client) Specialize(ctx context.Context, req *fission.FunctionSpecializeRequest) error {
_, err := sendRequest(ctx, c.httpClient, req, c.getSpecializeUrl()) _, err := sendRequest(c.logger, ctx, c.httpClient, req, c.getSpecializeUrl())
return err return err
} }
func (c *Client) Fetch(ctx context.Context, fr *fission.FunctionFetchRequest) error { func (c *Client) Fetch(ctx context.Context, fr *fission.FunctionFetchRequest) error {
_, err := sendRequest(ctx, c.httpClient, fr, c.getFetchUrl()) _, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getFetchUrl())
return err return err
} }
func (c *Client) Upload(ctx context.Context, fr *fission.ArchiveUploadRequest) (*fission.ArchiveUploadResponse, error) { func (c *Client) Upload(ctx context.Context, fr *fission.ArchiveUploadRequest) (*fission.ArchiveUploadResponse, error) {
body, err := sendRequest(ctx, c.httpClient, fr, c.getUploadUrl()) body, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getUploadUrl())
uploadResp := fission.ArchiveUploadResponse{} uploadResp := fission.ArchiveUploadResponse{}
err = json.Unmarshal(body, &uploadResp) err = json.Unmarshal(body, &uploadResp)
@@ -66,7 +70,7 @@ func (c *Client) Upload(ctx context.Context, fr *fission.ArchiveUploadRequest) (
return &uploadResp, nil return &uploadResp, nil
} }
func sendRequest(ctx context.Context, httpClient *http.Client, req interface{}, url string) ([]byte, error) { func sendRequest(logger *zap.Logger, ctx context.Context, httpClient *http.Client, req interface{}, url string) ([]byte, error) {
body, err := json.Marshal(req) body, err := json.Marshal(req)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -82,7 +86,7 @@ func sendRequest(ctx context.Context, httpClient *http.Client, req interface{},
if resp.StatusCode == 200 { if resp.StatusCode == 200 {
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
log.Printf("Error reading response body: %v", err) logger.Error("error reading response body", zap.Error(err))
} }
resp.Body.Close() resp.Body.Close()
return body, err return body, err
@@ -92,7 +96,7 @@ func sendRequest(ctx context.Context, httpClient *http.Client, req interface{},
if i < maxRetries-1 { if i < maxRetries-1 {
time.Sleep(50 * time.Duration(2*i) * time.Millisecond) time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error specializing/fetching/uploading package (%v) with url %v, retrying", err, url) logger.Error("error specializing/fetching/uploading package, retrying", zap.Error(err), zap.String("url", url))
continue continue
} }
} }
environments/fetcher/cmd/main.go
+17 -11
View File
@@ -12,6 +12,8 @@ import (
"runtime/debug" "runtime/debug"
"syscall" "syscall"
"go.uber.org/zap"
"go.opencensus.io/exporter/jaeger" "go.opencensus.io/exporter/jaeger"
"go.opencensus.io/plugin/ochttp" "go.opencensus.io/plugin/ochttp"
"go.opencensus.io/trace" "go.opencensus.io/trace"
@@ -49,14 +51,18 @@ func registerTraceExporter(collectorEndpoint string) error {
// Usage: fetcher <shared volume path> // Usage: fetcher <shared volume path>
func main() { func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
// register signal handler for dumping stack trace. // register signal handler for dumping stack trace.
c := make(chan os.Signal, 1) c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGTERM) signal.Notify(c, syscall.SIGTERM)
go func() { go func() {
<-c <-c
log.Println("Received SIGTERM : Dumping stack trace") logger.Fatal("received SIGTERM")
dumpStackTrace()
os.Exit(1)
}() }()
flag.Usage = fetcherUsage flag.Usage = fetcherUsage
@@ -77,18 +83,18 @@ func main() {
if os.IsNotExist(err) { if os.IsNotExist(err) {
err = os.MkdirAll(dir, os.ModeDir|0700) err = os.MkdirAll(dir, os.ModeDir|0700)
if err != nil { if err != nil {
log.Fatalf("Error creating directory: %v", err) logger.Fatal("error creating directory", zap.Error(err), zap.String("directory", dir))
} }
} }
} }
if err := registerTraceExporter(*collectorEndpoint); err != nil { if err := registerTraceExporter(*collectorEndpoint); err != nil {
log.Fatalf("Could not register trace exporter: %v", err) logger.Fatal("could not register trace exporter", zap.Error(err), zap.String("collector_endpoint", *collectorEndpoint))
} }
f, err := fetcher.MakeFetcher(dir, *secretDir, *configDir) f, err := fetcher.MakeFetcher(logger, dir, *secretDir, *configDir)
if err != nil { if err != nil {
log.Fatalf("Error making fetcher: %v", err) logger.Fatal("error making fetcher", zap.Error(err))
} }
readyToServe := false readyToServe := false
@@ -100,13 +106,13 @@ func main() {
err := json.Unmarshal([]byte(*specializePayload), &specializeReq) err := json.Unmarshal([]byte(*specializePayload), &specializeReq)
if err != nil { if err != nil {
log.Fatalf("Error decoding specialize request: %v", err) logger.Fatal("error decoding specialize request", zap.Error(err))
} }
ctx := context.Background() ctx := context.Background()
err = f.SpecializePod(ctx, specializeReq.FetchReq, specializeReq.LoadReq) err = f.SpecializePod(ctx, specializeReq.FetchReq, specializeReq.LoadReq)
if err != nil { if err != nil {
log.Fatalf("Error specialing function poadt: %v", err) logger.Fatal("error specializing function pod", zap.Error(err))
} }
readyToServe = true readyToServe = true
@@ -129,12 +135,12 @@ func main() {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
}) })
log.Println("Fetcher ready to receive requests") logger.Info("fetcher ready to receive requests")
http.ListenAndServe(":8000", &ochttp.Handler{ http.ListenAndServe(":8000", &ochttp.Handler{
Handler: mux, Handler: mux,
}) })
} }
func fetcherUsage() { func fetcherUsage() {
fmt.Printf("Usage: fetcher [-specialize-on-startup] [-specialize-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path> \n") fmt.Println("Usage: fetcher [-specialize-on-startup] [-specialize-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path>")
} }
environments/fetcher/fetcher.go
+144 -93
View File
@@ -9,7 +9,6 @@ import (
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"log"
"net" "net"
"net/http" "net/http"
"net/url" "net/url"
@@ -17,6 +16,8 @@ import (
"path/filepath" "path/filepath"
"time" "time"
"go.uber.org/zap"
"github.com/mholt/archiver" "github.com/mholt/archiver"
"github.com/pkg/errors" "github.com/pkg/errors"
uuid "github.com/satori/go.uuid" uuid "github.com/satori/go.uuid"
@@ -33,6 +34,7 @@ import (
type ( type (
Fetcher struct { Fetcher struct {
logger *zap.Logger
sharedVolumePath string sharedVolumePath string
sharedSecretPath string sharedSecretPath string
sharedConfigPath string sharedConfigPath string
@@ -42,23 +44,31 @@ type (
} }
) )
func makeVolumeDir(dirPath string) { func makeVolumeDir(dirPath string) error {
err := os.MkdirAll(dirPath, os.ModeDir|0700) return os.MkdirAll(dirPath, os.ModeDir|0700)
if err != nil {
log.Fatalf("Error creating %v: %v", dirPath, err)
}
} }
func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) { func MakeFetcher(logger *zap.Logger, sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) {
makeVolumeDir(sharedVolumePath) fLogger := logger.Named("fetcher")
makeVolumeDir(sharedSecretPath) err := makeVolumeDir(sharedVolumePath)
makeVolumeDir(sharedConfigPath) if err != nil {
fLogger.Fatal("error creating shared volume directory", zap.Error(err), zap.String("directory", sharedVolumePath))
}
err = makeVolumeDir(sharedSecretPath)
if err != nil {
fLogger.Fatal("error creating shared secret directory", zap.Error(err), zap.String("directory", sharedSecretPath))
}
err = makeVolumeDir(sharedConfigPath)
if err != nil {
fLogger.Fatal("error creating shared config directory", zap.Error(err), zap.String("directory", sharedConfigPath))
}
fissionClient, kubeClient, _, err := crd.MakeFissionClient() fissionClient, kubeClient, _, err := crd.MakeFissionClient()
if err != nil { if err != nil {
return nil, err return nil, errors.Wrap(err, "error making the fission / kube client")
} }
return &Fetcher{ return &Fetcher{
logger: fLogger,
sharedVolumePath: sharedVolumePath, sharedVolumePath: sharedVolumePath,
sharedSecretPath: sharedSecretPath, sharedSecretPath: sharedSecretPath,
sharedConfigPath: sharedConfigPath, sharedConfigPath: sharedConfigPath,
@@ -138,9 +148,7 @@ func writeSecretOrConfigMap(dataMap map[string][]byte, dirPath string) error {
writeFilePath := filepath.Join(dirPath, key) writeFilePath := filepath.Join(dirPath, key)
err := ioutil.WriteFile(writeFilePath, val, 0600) err := ioutil.WriteFile(writeFilePath, val, 0600)
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to write file %v: %v", writeFilePath, err) return errors.Wrapf(err, "Failed to write file %s", writeFilePath)
log.Printf(e)
return errors.New(e)
} }
} }
return nil return nil
@@ -160,38 +168,40 @@ func (fetcher *Fetcher) FetchHandler(w http.ResponseWriter, r *http.Request) {
startTime := time.Now() startTime := time.Now()
defer func() { defer func() {
elapsed := time.Since(startTime) elapsed := time.Since(startTime)
log.Printf("elapsed time in fetch request = %v", elapsed) fetcher.logger.Info("fetch request done", zap.Duration("elapsed_time", elapsed))
}() }()
// parse request // parse request
body, err := ioutil.ReadAll(r.Body) body, err := ioutil.ReadAll(r.Body)
if err != nil { if err != nil {
log.Printf("Error reading request body") fetcher.logger.Error("error reading request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
} }
var req fission.FunctionFetchRequest var req fission.FunctionFetchRequest
err = json.Unmarshal(body, &req) err = json.Unmarshal(body, &req)
if err != nil { if err != nil {
log.Printf("Error reading request body: %v", err) fetcher.logger.Error("error parsing request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusBadRequest) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
code, err := fetcher.Fetch(r.Context(), req) code, err := fetcher.Fetch(r.Context(), req)
if err != nil { if err != nil {
fetcher.logger.Error("error fetching", zap.Error(err))
http.Error(w, err.Error(), code) http.Error(w, err.Error(), code)
return return
} }
log.Printf("Checking secrets/cfgmaps") fetcher.logger.Info("checking secrets/cfgmaps")
code, err = fetcher.FetchSecretsAndCfgMaps(req.Secrets, req.ConfigMaps) code, err = fetcher.FetchSecretsAndCfgMaps(req.Secrets, req.ConfigMaps)
if err != nil { if err != nil {
fetcher.logger.Error("error fetching secrets and config maps", zap.Error(err))
http.Error(w, err.Error(), code) http.Error(w, err.Error(), code)
return return
} }
log.Printf("Completed fetch request") fetcher.logger.Info("completed fetch request")
// all done // all done
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
} }
@@ -205,22 +215,21 @@ func (fetcher *Fetcher) SpecializeHandler(w http.ResponseWriter, r *http.Request
// parse request // parse request
body, err := ioutil.ReadAll(r.Body) body, err := ioutil.ReadAll(r.Body)
if err != nil { if err != nil {
log.Printf("Error reading request body") fetcher.logger.Error("error reading request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
} }
var req fission.FunctionSpecializeRequest var req fission.FunctionSpecializeRequest
err = json.Unmarshal(body, &req) err = json.Unmarshal(body, &req)
if err != nil { if err != nil {
log.Printf("Error reading request body: %v", err) fetcher.logger.Error("error parsing request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusBadRequest) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
//log.Printf("fetcher received fetch request and started downloading: %v", req)
err = fetcher.SpecializePod(r.Context(), req.FetchReq, req.LoadReq) err = fetcher.SpecializePod(r.Context(), req.FetchReq, req.LoadReq)
if err != nil { if err != nil {
fetcher.logger.Error("error specialing pod", zap.Error(err))
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
return return
} }
@@ -234,14 +243,16 @@ func (fetcher *Fetcher) SpecializeHandler(w http.ResponseWriter, r *http.Request
func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequest) (int, error) { func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequest) (int, error) {
// check that the requested filename is not an empty string and error out if so // check that the requested filename is not an empty string and error out if so
if len(req.Filename) == 0 { if len(req.Filename) == 0 {
e := fmt.Sprintf("Fetch request received for an empty file name, request: %v", req) e := "fetch request received for an empty file name"
log.Printf(e) fetcher.logger.Error(e, zap.Any("request", req))
return http.StatusBadRequest, errors.New(e) return http.StatusBadRequest, errors.New(fmt.Sprintf("%s, request: %v", e, req))
} }
// verify first if the file already exists. // verify first if the file already exists.
if _, err := os.Stat(filepath.Join(fetcher.sharedVolumePath, req.Filename)); err == nil { if _, err := os.Stat(filepath.Join(fetcher.sharedVolumePath, req.Filename)); err == nil {
log.Printf("Requested file: %s already exists at %s. Skipping fetch", req.Filename, fetcher.sharedVolumePath) fetcher.logger.Info("requested file already exists at shared volume - skipping fetch",
zap.String("requested_file", req.Filename),
zap.String("shared_volume_path", fetcher.sharedVolumePath))
return http.StatusOK, nil return http.StatusOK, nil
} }
@@ -252,17 +263,19 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
// fetch the file and save it to the tmp path // fetch the file and save it to the tmp path
err := downloadUrl(ctx, fetcher.httpClient, req.Url, tmpPath) err := downloadUrl(ctx, fetcher.httpClient, req.Url, tmpPath)
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err) e := "failed to download url"
log.Printf(e) fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url))
return http.StatusBadRequest, errors.New(e) return http.StatusBadRequest, errors.Wrapf(err, "%s: %s", e, req.Url)
} }
} else { } else {
// get pkg // get pkg
pkg, err := fetcher.fissionClient.Packages(req.Package.Namespace).Get(req.Package.Name) pkg, err := fetcher.fissionClient.Packages(req.Package.Namespace).Get(req.Package.Name)
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to get package: %v", err) e := "failed to get package"
log.Printf(e) fetcher.logger.Error(e,
return http.StatusInternalServerError, errors.New(e) zap.String("package_name", req.Package.Name),
zap.String("package_namespace", req.Package.Namespace))
return http.StatusInternalServerError, errors.Wrap(err, e)
} }
var archive *fission.Archive var archive *fission.Archive
@@ -274,9 +287,12 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
// we hit this "Get : unsupported protocol scheme "" error. // we hit this "Get : unsupported protocol scheme "" error.
// it may be useful to the user if we can send a more meaningful error in such a scenario. // it may be useful to the user if we can send a more meaningful error in such a scenario.
if pkg.Status.BuildStatus != fission.BuildStatusSucceeded && pkg.Status.BuildStatus != fission.BuildStatusNone { if pkg.Status.BuildStatus != fission.BuildStatusSucceeded && pkg.Status.BuildStatus != fission.BuildStatusNone {
e := fmt.Sprintf("Build status for the function's pkg : %s.%s is : %s, can't fetch deployment", pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Status.BuildStatus) e := fmt.Sprintf("cannot fetch deployment: package build status was not %q", fission.BuildStatusSucceeded)
log.Printf(e) fetcher.logger.Error(e,
return http.StatusInternalServerError, errors.New(e) zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namespace", pkg.Metadata.Namespace),
zap.Any("package_build_status", pkg.Status.BuildStatus))
return http.StatusInternalServerError, errors.New(fmt.Sprintf("%s: pkg %s.%s has a status of %s", e, pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Status.BuildStatus))
} }
archive = &pkg.Spec.Deployment archive = &pkg.Spec.Deployment
} }
@@ -285,31 +301,31 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
// write pkg.Literal into tmpPath // write pkg.Literal into tmpPath
err = ioutil.WriteFile(tmpPath, archive.Literal, 0600) err = ioutil.WriteFile(tmpPath, archive.Literal, 0600)
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to write file %v: %v", tmpPath, err) e := "failed to write file"
log.Printf(e) fetcher.logger.Error(e, zap.Error(err), zap.String("location", tmpPath))
return http.StatusInternalServerError, errors.New(e) return http.StatusInternalServerError, errors.Wrapf(err, "%s %s", e, tmpPath)
} }
} else { } else {
// download and verify // download and verify
err := downloadUrl(ctx, fetcher.httpClient, archive.URL, tmpPath) err := downloadUrl(ctx, fetcher.httpClient, archive.URL, tmpPath)
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err) e := "failed to download url"
log.Printf(e) fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url))
return http.StatusBadRequest, errors.New(e) return http.StatusBadRequest, errors.Wrapf(err, "%s %s", e, req.Url)
} }
checksum, err := getChecksum(tmpPath) checksum, err := getChecksum(tmpPath)
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to get checksum: %v", err) e := "failed to get checksum"
log.Printf(e) fetcher.logger.Error(e, zap.Error(err))
return http.StatusBadRequest, errors.New(e) return http.StatusBadRequest, errors.Wrap(err, e)
} }
err = verifyChecksum(checksum, &archive.Checksum) err = verifyChecksum(checksum, &archive.Checksum)
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to verify checksum: %v", err) e := "failed to verify checksum"
log.Printf(e) fetcher.logger.Error(e, zap.Error(err))
return http.StatusBadRequest, errors.New(e) return http.StatusBadRequest, errors.Wrap(err, e)
} }
} }
} }
@@ -319,7 +335,10 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
tmpUnarchivePath := filepath.Join(fetcher.sharedVolumePath, uuid.NewV4().String()) tmpUnarchivePath := filepath.Join(fetcher.sharedVolumePath, uuid.NewV4().String())
err := fetcher.unarchive(tmpPath, tmpUnarchivePath) err := fetcher.unarchive(tmpPath, tmpUnarchivePath)
if err != nil { if err != nil {
log.Println(err.Error()) fetcher.logger.Error("error unarchiving",
zap.Error(err),
zap.String("archive_location", tmpPath),
zap.String("target_location", tmpUnarchivePath))
return http.StatusInternalServerError, err return http.StatusInternalServerError, err
} }
@@ -327,13 +346,17 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
} }
// move tmp file to requested filename // move tmp file to requested filename
err := fetcher.rename(tmpPath, filepath.Join(fetcher.sharedVolumePath, req.Filename)) renamePath := filepath.Join(fetcher.sharedVolumePath, req.Filename)
err := fetcher.rename(tmpPath, renamePath)
if err != nil { if err != nil {
log.Println(err.Error()) fetcher.logger.Error("error renaming file",
zap.Error(err),
zap.String("original_path", tmpPath),
zap.String("rename_path", renamePath))
return http.StatusInternalServerError, err return http.StatusInternalServerError, err
} }
log.Printf("Successfully placed at %v", filepath.Join(fetcher.sharedVolumePath, req.Filename)) fetcher.logger.Info("successfully placed", zap.String("location", renamePath))
return http.StatusOK, nil return http.StatusOK, nil
} }
@@ -345,13 +368,17 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
data, err := fetcher.kubeClient.CoreV1().Secrets(secret.Namespace).Get(secret.Name, metav1.GetOptions{}) data, err := fetcher.kubeClient.CoreV1().Secrets(secret.Namespace).Get(secret.Name, metav1.GetOptions{})
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to get secret from kubeapi: %v", err) e := "error getting secret from kubeapi"
log.Printf(e)
httpCode := http.StatusInternalServerError httpCode := http.StatusInternalServerError
if k8serr.IsNotFound(err) { if k8serr.IsNotFound(err) {
httpCode = http.StatusNotFound httpCode = http.StatusNotFound
e = "secret was not found in kubeapi"
} }
fetcher.logger.Error(e,
zap.Error(err),
zap.String("secret_name", secret.Name),
zap.String("secret_namespace", secret.Namespace))
return httpCode, errors.New(e) return httpCode, errors.New(e)
} }
@@ -360,12 +387,21 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
secretDir := filepath.Join(fetcher.sharedSecretPath, secretPath) secretDir := filepath.Join(fetcher.sharedSecretPath, secretPath)
err = os.MkdirAll(secretDir, os.ModeDir|0644) err = os.MkdirAll(secretDir, os.ModeDir|0644)
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to create directory %v: %v", secretDir, err) e := "failed to create directory for secret"
log.Printf(e) fetcher.logger.Error(e,
return http.StatusInternalServerError, errors.New(e) zap.Error(err),
zap.String("directory", secretDir),
zap.String("secret_name", secret.Name),
zap.String("secret_namespace", secret.Namespace))
return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, secretDir)
} }
err = writeSecretOrConfigMap(data.Data, secretDir) err = writeSecretOrConfigMap(data.Data, secretDir)
if err != nil { if err != nil {
fetcher.logger.Error("failed to write secret to file location",
zap.Error(err),
zap.String("location", secretDir),
zap.String("secret_name", secret.Name),
zap.String("secret_namespace", secret.Namespace))
return http.StatusInternalServerError, err return http.StatusInternalServerError, err
} }
} }
@@ -376,13 +412,17 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
data, err := fetcher.kubeClient.CoreV1().ConfigMaps(config.Namespace).Get(config.Name, metav1.GetOptions{}) data, err := fetcher.kubeClient.CoreV1().ConfigMaps(config.Namespace).Get(config.Name, metav1.GetOptions{})
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to get configmap from kubeapi: %v", err) e := "error getting configmap from kubeapi"
log.Printf(e)
httpCode := http.StatusInternalServerError httpCode := http.StatusInternalServerError
if k8serr.IsNotFound(err) { if k8serr.IsNotFound(err) {
httpCode = http.StatusNotFound httpCode = http.StatusNotFound
e = "configmap was not found in kubeapi"
} }
fetcher.logger.Error(e,
zap.Error(err),
zap.String("config_map_name", config.Name),
zap.String("config_map_namespace", config.Namespace))
return httpCode, errors.New(e) return httpCode, errors.New(e)
} }
@@ -391,9 +431,13 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
configDir := filepath.Join(fetcher.sharedConfigPath, configPath) configDir := filepath.Join(fetcher.sharedConfigPath, configPath)
err = os.MkdirAll(configDir, os.ModeDir|0644) err = os.MkdirAll(configDir, os.ModeDir|0644)
if err != nil { if err != nil {
e := fmt.Sprintf("Failed to create directory %v: %v", configDir, err) e := "failed to create directory for configmap"
log.Printf(e) fetcher.logger.Error(e,
return http.StatusInternalServerError, errors.New(e) zap.Error(err),
zap.String("directory", configDir),
zap.String("config_map_name", config.Name),
zap.String("config_map_namespace", config.Namespace))
return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, configDir)
} }
configMap := make(map[string][]byte) configMap := make(map[string][]byte)
for key, val := range data.Data { for key, val := range data.Data {
@@ -401,6 +445,11 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
} }
err = writeSecretOrConfigMap(configMap, configDir) err = writeSecretOrConfigMap(configMap, configDir)
if err != nil { if err != nil {
fetcher.logger.Error("failed to write configmap to file location",
zap.Error(err),
zap.String("location", configDir),
zap.String("config_map_name", config.Name),
zap.String("config_map_namespace", config.Namespace))
return http.StatusInternalServerError, err return http.StatusInternalServerError, err
} }
} }
@@ -418,13 +467,13 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
startTime := time.Now() startTime := time.Now()
defer func() { defer func() {
elapsed := time.Since(startTime) elapsed := time.Since(startTime)
log.Printf("elapsed time in upload request = %v", elapsed) fetcher.logger.Info("upload request done", zap.Duration("elapsed_time", elapsed))
}() }()
// parse request // parse request
body, err := ioutil.ReadAll(r.Body) body, err := ioutil.ReadAll(r.Body)
if err != nil { if err != nil {
log.Printf("Error reading request body") fetcher.logger.Error("error reading request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
} }
@@ -432,11 +481,11 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
var req fission.ArchiveUploadRequest var req fission.ArchiveUploadRequest
err = json.Unmarshal(body, &req) err = json.Unmarshal(body, &req)
if err != nil { if err != nil {
log.Printf("Error reading request body: %v", err) fetcher.logger.Error("error parsing request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusBadRequest) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
log.Printf("fetcher received upload request: %v", req) fetcher.logger.Info("fetcher received upload request", zap.Any("request", req))
zipFilename := req.Filename + ".zip" zipFilename := req.Filename + ".zip"
srcFilepath := filepath.Join(fetcher.sharedVolumePath, req.Filename) srcFilepath := filepath.Join(fetcher.sharedVolumePath, req.Filename)
@@ -445,37 +494,37 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
if req.ArchivePackage { if req.ArchivePackage {
err = fetcher.archive(srcFilepath, dstFilepath) err = fetcher.archive(srcFilepath, dstFilepath)
if err != nil { if err != nil {
e := fmt.Sprintf("Error archiving zip file: %v", err) e := "error archiving zip file"
log.Println(e) fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath))
http.Error(w, e, http.StatusInternalServerError) http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return return
} }
} else { } else {
err = os.Rename(srcFilepath, dstFilepath) err = os.Rename(srcFilepath, dstFilepath)
if err != nil { if err != nil {
e := fmt.Sprintf("Error renaming the archive: %v", err) e := "error renaming the archive"
log.Println(e) fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath))
http.Error(w, e, http.StatusInternalServerError) http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return return
} }
} }
log.Println("Starting upload...") fetcher.logger.Info("starting upload...")
ssClient := storageSvcClient.MakeClient(req.StorageSvcUrl) ssClient := storageSvcClient.MakeClient(req.StorageSvcUrl)
fileID, err := ssClient.Upload(r.Context(), dstFilepath, nil) fileID, err := ssClient.Upload(r.Context(), dstFilepath, nil)
if err != nil { if err != nil {
e := fmt.Sprintf("Error uploading zip file: %v", err) e := "error uploading zip file"
log.Println(e) fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath))
http.Error(w, e, http.StatusInternalServerError) http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return return
} }
sum, err := getChecksum(dstFilepath) sum, err := getChecksum(dstFilepath)
if err != nil { if err != nil {
e := fmt.Sprintf("Error calculating checksum of zip file: %v", err) e := "error calculating checksum of zip file"
log.Println(e) fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath))
http.Error(w, e, http.StatusInternalServerError) http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return return
} }
@@ -486,13 +535,13 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
rBody, err := json.Marshal(resp) rBody, err := json.Marshal(resp)
if err != nil { if err != nil {
e := fmt.Sprintf("Error encoding upload response: %v", err) e := "error encoding upload response"
log.Println(e) fetcher.logger.Error(e, zap.Error(err))
http.Error(w, e, http.StatusInternalServerError) http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return return
} }
log.Println("Completed upload request") fetcher.logger.Info("completed upload request")
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
w.Write(rBody) w.Write(rBody)
@@ -501,7 +550,7 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
func (fetcher *Fetcher) rename(src string, dst string) error { func (fetcher *Fetcher) rename(src string, dst string) error {
err := os.Rename(src, dst) err := os.Rename(src, dst)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("Failed to move file: %v", err)) return errors.Wrap(err, "failed to move file")
} }
return nil return nil
} }
@@ -512,7 +561,7 @@ func (fetcher *Fetcher) archive(src string, dst string) error {
var files []string var files []string
target, err := os.Stat(src) target, err := os.Stat(src)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("Failed to zip file: %v", err)) return errors.Wrap(err, "failed to zip file")
} }
if target.IsDir() { if target.IsDir() {
// list all // list all
@@ -530,7 +579,7 @@ func (fetcher *Fetcher) archive(src string, dst string) error {
func (fetcher *Fetcher) unarchive(src string, dst string) error { func (fetcher *Fetcher) unarchive(src string, dst string) error {
err := archiver.Zip.Open(src, dst) err := archiver.Zip.Open(src, dst)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("Failed to unzip file: %v", err)) return errors.Wrap(err, "failed to unzip file")
} }
return nil return nil
} }
@@ -539,17 +588,17 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func
startTime := time.Now() startTime := time.Now()
defer func() { defer func() {
elapsed := time.Since(startTime) elapsed := time.Since(startTime)
log.Printf("Elapsed time in fetch request = %v", elapsed) fetcher.logger.Info("specialize request done", zap.Duration("elapsed_time", elapsed))
}() }()
_, err := fetcher.Fetch(ctx, fetchReq) _, err := fetcher.Fetch(ctx, fetchReq)
if err != nil { if err != nil {
return errors.Wrap(err, "Error fetching deploy package") return errors.Wrap(err, "error fetching deploy package")
} }
_, err = fetcher.FetchSecretsAndCfgMaps(fetchReq.Secrets, fetchReq.ConfigMaps) _, err = fetcher.FetchSecretsAndCfgMaps(fetchReq.Secrets, fetchReq.ConfigMaps)
if err != nil { if err != nil {
return errors.Wrap(err, "Error fetching secrets/configmaps") return errors.Wrap(err, "error fetching secrets/configmaps")
} }
// Specialize the pod // Specialize the pod
@@ -561,17 +610,19 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func
loadPayload, err := json.Marshal(loadReq) loadPayload, err := json.Marshal(loadReq)
if err != nil { if err != nil {
return errors.Wrap(err, "Error encoding load request") return errors.Wrap(err, "error encoding load request")
} }
if loadReq.EnvVersion >= 2 { if loadReq.EnvVersion >= 2 {
contentType = "application/json" contentType = "application/json"
specializeURL = "http://localhost:8888/v2/specialize" specializeURL = "http://localhost:8888/v2/specialize"
reader = bytes.NewReader(loadPayload) reader = bytes.NewReader(loadPayload)
fetcher.logger.Info("calling environment v2 specialization endpoint")
} else { } else {
contentType = "text/plain" contentType = "text/plain"
specializeURL = "http://localhost:8888/specialize" specializeURL = "http://localhost:8888/specialize"
reader = bytes.NewReader([]byte{}) reader = bytes.NewReader([]byte{})
fetcher.logger.Info("calling environment v1 specialization endpoint")
} }
for i := 0; i < maxRetries; i++ { for i := 0; i < maxRetries; i++ {
@@ -588,7 +639,7 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func
if netErr.Op == "dial" { if netErr.Op == "dial" {
if i < maxRetries-1 { if i < maxRetries-1 {
time.Sleep(500 * time.Duration(2*i) * time.Millisecond) time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error connecting to pod (%v), retrying", netErr) fetcher.logger.Error("error connecting to function environment pod for specialization request, retrying", zap.Error(netErr))
continue continue
} }
} }
@@ -599,8 +650,8 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func
err = fission.MakeErrorFromHTTP(resp) err = fission.MakeErrorFromHTTP(resp)
} }
return errors.Wrap(err, "Error specializing function pod") return errors.Wrap(err, "error specializing function pod")
} }
return errors.Wrap(err, fmt.Sprintf("Error specializing function pod after %v times", maxRetries)) return errors.Wrapf(err, "error specializing function pod after %v times", maxRetries)
} }
environments/go/server.go
+95 -76
View File
@@ -10,6 +10,7 @@ import (
"plugin" "plugin"
"github.com/pkg/errors" "github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/environments/go/context" "github.com/fission/fission/environments/go/context"
) )
@@ -39,7 +40,7 @@ type (
var userFunc http.HandlerFunc var userFunc http.HandlerFunc
func loadPlugin(codePath, entrypoint string) (http.HandlerFunc, error) { func loadPlugin(logger *zap.Logger, codePath, entrypoint string) (http.HandlerFunc, error) {
// if codepath's a directory, load the file inside it // if codepath's a directory, load the file inside it
info, err := os.Stat(codePath) info, err := os.Stat(codePath)
@@ -58,7 +59,7 @@ func loadPlugin(codePath, entrypoint string) (http.HandlerFunc, error) {
codePath = filepath.Join(codePath, fi.Name()) codePath = filepath.Join(codePath, fi.Name())
} }
log.Printf("loading plugin from %v\n", codePath) logger.Info("loading plugin", zap.String("location", codePath))
p, err := plugin.Open(codePath) p, err := plugin.Open(codePath)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "error loading plugin") return nil, errors.Wrap(err, "error loading plugin")
@@ -85,87 +86,99 @@ func loadPlugin(codePath, entrypoint string) (http.HandlerFunc, error) {
} }
} }
func specializeHandler(w http.ResponseWriter, r *http.Request) { func specializeHandler(logger *zap.Logger) func(http.ResponseWriter, *http.Request) {
if userFunc != nil { return func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest) if userFunc != nil {
w.Write([]byte("Not a generic container")) w.WriteHeader(http.StatusBadRequest)
return w.Write([]byte("Not a generic container"))
}
_, err := os.Stat(CODE_PATH)
if err != nil {
if os.IsNotExist(err) {
w.WriteHeader(http.StatusNotFound)
log.Printf(err.Error())
w.Write([]byte(CODE_PATH + ": not found"))
return
} else {
err = errors.Wrap(err, "unknown error")
log.Printf(err.Error())
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return return
} }
}
log.Println("Specializing ...") _, err := os.Stat(CODE_PATH)
userFunc, err = loadPlugin(CODE_PATH, "Handler") if err != nil {
if err != nil { if os.IsNotExist(err) {
err = errors.Wrap(err, "error specializing function") w.WriteHeader(http.StatusNotFound)
log.Printf(err.Error()) logger.Error("code path does not exist",
w.WriteHeader(http.StatusInternalServerError) zap.Error(err),
w.Write([]byte(err.Error())) zap.String("code_path", CODE_PATH))
return w.Write([]byte(CODE_PATH + ": not found"))
return
} else {
logger.Error("unknown error looking for code path",
zap.Error(err),
zap.String("code_path", CODE_PATH))
err = errors.Wrap(err, "unknown error")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
}
logger.Info("specializing ...")
userFunc, err = loadPlugin(logger, CODE_PATH, "Handler")
if err != nil {
e := "error specializing function"
logger.Error(e, zap.Error(err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(errors.Wrap(err, e).Error()))
return
}
logger.Info("done")
} }
log.Println("Done")
} }
func specializeHandlerV2(w http.ResponseWriter, r *http.Request) { func specializeHandlerV2(logger *zap.Logger) func(http.ResponseWriter, *http.Request) {
if userFunc != nil { return func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusBadRequest) if userFunc != nil {
w.Write([]byte("Not a generic container")) w.WriteHeader(http.StatusBadRequest)
return w.Write([]byte("Not a generic container"))
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Println(err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
var loadreq FunctionLoadRequest
err = json.Unmarshal(body, &loadreq)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
_, err = os.Stat(loadreq.FilePath)
if err != nil {
if os.IsNotExist(err) {
w.WriteHeader(http.StatusNotFound)
log.Printf(err.Error())
w.Write([]byte(CODE_PATH + ": not found"))
return
} else {
err = errors.Wrap(err, "unknown error")
log.Printf(err.Error())
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return return
} }
}
log.Println("Specializing ...") body, err := ioutil.ReadAll(r.Body)
userFunc, err = loadPlugin(loadreq.FilePath, loadreq.FunctionName) if err != nil {
if err != nil { logger.Error("error reading request body", zap.Error(err))
err = errors.Wrap(err, "error specializing function") w.WriteHeader(http.StatusInternalServerError)
log.Printf(err.Error()) return
w.WriteHeader(http.StatusInternalServerError) }
w.Write([]byte(err.Error())) var loadreq FunctionLoadRequest
return err = json.Unmarshal(body, &loadreq)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
_, err = os.Stat(loadreq.FilePath)
if err != nil {
if os.IsNotExist(err) {
logger.Error("code path does not exist",
zap.Error(err),
zap.String("code_path", CODE_PATH))
w.WriteHeader(http.StatusNotFound)
w.Write([]byte(CODE_PATH + ": not found"))
return
} else {
logger.Error("unknown error looking for code path",
zap.Error(err),
zap.String("code_path", CODE_PATH))
err = errors.Wrap(err, "unknown error")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
}
logger.Info("specializing ...")
userFunc, err = loadPlugin(logger, loadreq.FilePath, loadreq.FunctionName)
if err != nil {
e := "error specializing function"
logger.Error(e, zap.Error(err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(errors.Wrap(err, e).Error()))
return
}
logger.Info("done")
} }
log.Println("Done")
} }
func readinessProbeHandler(w http.ResponseWriter, r *http.Request) { func readinessProbeHandler(w http.ResponseWriter, r *http.Request) {
@@ -173,9 +186,15 @@ func readinessProbeHandler(w http.ResponseWriter, r *http.Request) {
} }
func main() { func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
http.HandleFunc("/healthz", readinessProbeHandler) http.HandleFunc("/healthz", readinessProbeHandler)
http.HandleFunc("/specialize", specializeHandler) http.HandleFunc("/specialize", specializeHandler(logger.Named("specialize_handler")))
http.HandleFunc("/v2/specialize", specializeHandlerV2) http.HandleFunc("/v2/specialize", specializeHandlerV2(logger.Named("specialize_v2_handler")))
// Generic route -- all http requests go to the user function. // Generic route -- all http requests go to the user function.
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
@@ -187,6 +206,6 @@ func main() {
userFunc(w, r) userFunc(w, r)
}) })
log.Println("Listening on 8888 ...") logger.Info("listening on 8888 ...")
http.ListenAndServe(":8888", nil) http.ListenAndServe(":8888", nil)
} }
executor/api.go
+20 -8
View File
@@ -21,12 +21,12 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"strings" "strings"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"go.opencensus.io/plugin/ochttp" "go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
@@ -50,7 +50,10 @@ func (executor *Executor) getServiceForFunctionApi(w http.ResponseWriter, r *htt
serviceName, err := executor.getServiceForFunction(r.Context(), &m) serviceName, err := executor.getServiceForFunction(r.Context(), &m)
if err != nil { if err != nil {
code, msg := fission.GetHTTPError(err) code, msg := fission.GetHTTPError(err)
log.Printf("Error: %v: %v", code, msg) executor.logger.Error("error getting service for function",
zap.Error(err),
zap.String("function", m.Name),
zap.String("fission_http_error", msg))
http.Error(w, msg, code) http.Error(w, msg, code)
return return
} }
@@ -69,14 +72,19 @@ func (executor *Executor) getServiceForFunctionApi(w http.ResponseWriter, r *htt
// invalidates the cache entry if the pod address was cached. // invalidates the cache entry if the pod address was cached.
func (executor *Executor) getServiceForFunction(ctx context.Context, m *metav1.ObjectMeta) (string, error) { func (executor *Executor) getServiceForFunction(ctx context.Context, m *metav1.ObjectMeta) (string, error) {
// Check function -> svc cache // Check function -> svc cache
log.Printf("[%v] Checking for cached function service", m.Name) executor.logger.Info("checking for cached function service",
zap.String("function_name", m.Name),
zap.String("function_namespace", m.Namespace))
fsvc, err := executor.fsCache.GetByFunction(m) fsvc, err := executor.fsCache.GetByFunction(m)
if err == nil { if err == nil {
if executor.isValidAddress(fsvc) { if executor.isValidAddress(fsvc) {
// Cached, return svc address // Cached, return svc address
return fsvc.Address, nil return fsvc.Address, nil
} else { } else {
log.Printf("[%v] Deleting cache entry for invalid address : %s", m.Name, fsvc.Address) executor.logger.Info("deleting cache entry for invalid address",
zap.String("function_name", m.Name),
zap.String("function_namespace", m.Namespace),
zap.String("address", fsvc.Address))
executor.fsCache.DeleteEntry(fsvc) executor.fsCache.DeleteEntry(fsvc)
} }
} }
@@ -98,6 +106,7 @@ func (executor *Executor) getServiceForFunction(ctx context.Context, m *metav1.O
func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) { func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body) body, err := ioutil.ReadAll(r.Body)
if err != nil { if err != nil {
executor.logger.Error("failed to read tap service request", zap.Error(err))
http.Error(w, "Failed to read request", http.StatusInternalServerError) http.Error(w, "Failed to read request", http.StatusInternalServerError)
return return
} }
@@ -106,7 +115,10 @@ func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) {
err = executor.fsCache.TouchByAddress(svcHost) err = executor.fsCache.TouchByAddress(svcHost)
if err != nil { if err != nil {
log.Printf("funcSvc tap error: %v", err) executor.logger.Error("error tapping function service",
zap.Error(err),
zap.String("service", svcName),
zap.String("host", svcHost))
http.Error(w, "Not found", http.StatusNotFound) http.Error(w, "Not found", http.StatusNotFound)
return return
} }
@@ -123,15 +135,15 @@ func (executor *Executor) Serve(port int) {
r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST") r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST")
r.HandleFunc("/healthz", executor.healthHandler).Methods("GET") r.HandleFunc("/healthz", executor.healthHandler).Methods("GET")
address := fmt.Sprintf(":%v", port) address := fmt.Sprintf(":%v", port)
log.Printf("starting executor at port %v", port) executor.logger.Info("starting executor", zap.Int("port", port))
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
executor.ndm.Run(ctx) executor.ndm.Run(ctx)
executor.gpm.Run(ctx) executor.gpm.Run(ctx)
r.Use(fission.LoggingMiddleware) r.Use(fission.LoggingMiddleware(executor.logger))
err := http.ListenAndServe(address, &ochttp.Handler{ err := http.ListenAndServe(address, &ochttp.Handler{
Handler: r, Handler: r,
// Propagation: &b3.HTTPFormat{}, // Propagation: &b3.HTTPFormat{},
}) })
log.Fatal(err) executor.logger.Fatal("done listening", zap.Error(err))
} }
executor/client/client.go
+10 -8
View File
@@ -21,13 +21,14 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"net/url" "net/url"
"strings" "strings"
"time" "time"
"github.com/pkg/errors"
"go.opencensus.io/plugin/ochttp" "go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
"golang.org/x/net/context/ctxhttp" "golang.org/x/net/context/ctxhttp"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,14 +36,16 @@ import (
) )
type Client struct { type Client struct {
logger *zap.Logger
executorUrl string executorUrl string
tappedByUrl map[string]bool tappedByUrl map[string]bool
requestChan chan string requestChan chan string
httpClient *http.Client httpClient *http.Client
} }
func MakeClient(executorUrl string) *Client { func MakeClient(logger *zap.Logger, executorUrl string) *Client {
c := &Client{ c := &Client{
logger: logger.Named("executor_client"),
executorUrl: strings.TrimSuffix(executorUrl, "/"), executorUrl: strings.TrimSuffix(executorUrl, "/"),
tappedByUrl: make(map[string]bool), tappedByUrl: make(map[string]bool),
requestChan: make(chan string), requestChan: make(chan string),
@@ -59,12 +62,12 @@ func (c *Client) GetServiceForFunction(ctx context.Context, metadata *metav1.Obj
body, err := json.Marshal(metadata) body, err := json.Marshal(metadata)
if err != nil { if err != nil {
return "", err return "", errors.Wrap(err, "could not marshal request body for getting service for function")
} }
resp, err := ctxhttp.Post(ctx, c.httpClient, executorUrl, "application/json", bytes.NewReader(body)) resp, err := ctxhttp.Post(ctx, c.httpClient, executorUrl, "application/json", bytes.NewReader(body))
if err != nil { if err != nil {
return "", err return "", errors.Wrap(err, "error posting to getting service for function")
} }
defer resp.Body.Close() defer resp.Body.Close()
@@ -74,8 +77,7 @@ func (c *Client) GetServiceForFunction(ctx context.Context, metadata *metav1.Obj
svcName, err := ioutil.ReadAll(resp.Body) svcName, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
log.Printf("Returning from ioutil read body") return "", errors.Wrap(err, "error reading response body from getting service for function")
return "", err
} }
return string(svcName), nil return string(svcName), nil
@@ -95,10 +97,10 @@ func (c *Client) service() {
for u := range urls { for u := range urls {
err := c._tapService(u) err := c._tapService(u)
if err != nil { if err != nil {
log.Printf("Error tapping function service address %v: %v", u, err) c.logger.Error("error tapping function service address", zap.Error(err), zap.String("address", u))
} }
} }
log.Printf("Tapped %v services in batch", len(urls)) c.logger.Info("tapped services in batch", zap.Int("service_count", len(urls)))
}() }()
} }
} }
executor/executor.go
+33 -18
View File
@@ -19,7 +19,6 @@ package executor
import ( import (
"context" "context"
"fmt" "fmt"
"log"
"net/http" "net/http"
"runtime/debug" "runtime/debug"
"strings" "strings"
@@ -29,6 +28,7 @@ import (
"github.com/dchest/uniuri" "github.com/dchest/uniuri"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
@@ -41,6 +41,8 @@ import (
type ( type (
Executor struct { Executor struct {
logger *zap.Logger
gpm *poolmgr.GenericPoolManager gpm *poolmgr.GenericPoolManager
ndm *newdeploy.NewDeploy ndm *newdeploy.NewDeploy
@@ -62,8 +64,9 @@ type (
} }
) )
func MakeExecutor(gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor { func MakeExecutor(logger *zap.Logger, gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor {
executor := &Executor{ executor := &Executor{
logger: logger.Named("executor"),
gpm: gpm, gpm: gpm,
ndm: ndm, ndm: ndm,
fissionClient: fissionClient, fissionClient: fissionClient,
@@ -111,7 +114,8 @@ func (executor *Executor) serveCreateFuncServices() {
} else { } else {
// There's an existing request for this function, wait for it to finish // There's an existing request for this function, wait for it to finish
go func() { go func() {
log.Printf("Waiting for concurrent request for the same function: %v", m) executor.logger.Info("waiting for concurrent request for the same function",
zap.Any("function", m))
wg.Wait() wg.Wait()
// get the function service from the cache // get the function service from the cache
@@ -121,7 +125,9 @@ func (executor *Executor) serveCreateFuncServices() {
// It normally happened if there are multiple requests are // It normally happened if there are multiple requests are
// waiting for the same function and executor failed to cre- // waiting for the same function and executor failed to cre-
// ate service for function. // ate service for function.
err = errors.Wrap(err, fmt.Sprintf("Error getting service for function %v in namespace %v", m.Name, m.Namespace)) err = errors.Wrapf(err, "error getting service for function",
zap.String("function_name", m.Name),
zap.String("function_namespace", m.Namespace))
req.respChan <- &createFuncServiceResponse{ req.respChan <- &createFuncServiceResponse{
funcSvc: fsvc, funcSvc: fsvc,
err: err, err: err,
@@ -140,7 +146,9 @@ func (executor *Executor) getFunctionExecutorType(meta *metav1.ObjectMeta) (fiss
} }
func (executor *Executor) createServiceForFunction(ctx context.Context, meta *metav1.ObjectMeta) (*fscache.FuncSvc, error) { func (executor *Executor) createServiceForFunction(ctx context.Context, meta *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
log.Printf("[%v] No cached function service found, creating one", meta.Name) executor.logger.Info("no cached function service found, creating one",
zap.String("function_name", meta.Name),
zap.String("function_namespace", meta.Namespace))
executorType, err := executor.getFunctionExecutorType(meta) executorType, err := executor.getFunctionExecutorType(meta)
if err != nil { if err != nil {
@@ -158,8 +166,12 @@ func (executor *Executor) createServiceForFunction(ctx context.Context, meta *me
} }
if fsvcErr != nil { if fsvcErr != nil {
fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%v] Error creating service for function", meta.Name)) e := "error creating service for function"
log.Print(fsvcErr) executor.logger.Error(e,
zap.Error(fsvcErr),
zap.String("function_name", meta.Name),
zap.String("function_namespace", meta.Namespace))
fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%s] %s", meta.Name, e))
} else if fsvc != nil { } else if fsvc != nil {
_, err = executor.fsCache.Add(*fsvc) _, err = executor.fsCache.Add(*fsvc)
if err != nil { if err != nil {
@@ -185,16 +197,18 @@ func dumpStackTrace() {
debug.PrintStack() debug.PrintStack()
} }
func serveMetric() { func serveMetric(logger *zap.Logger) {
// Expose the registered metrics via HTTP. // Expose the registered metrics via HTTP.
metricAddr := ":8080" metricAddr := ":8080"
http.Handle("/metrics", promhttp.Handler()) http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(metricAddr, nil)) err := http.ListenAndServe(metricAddr, nil)
logger.Fatal("done listening on metrics endpoint", zap.Error(err))
} }
// StartExecutor Starts executor and the executor components such as Poolmgr, // StartExecutor Starts executor and the executor components such as Poolmgr,
// deploymgr and potential future executor types // deploymgr and potential future executor types
func StartExecutor(fissionNamespace string, functionNamespace string, envBuilderNamespace string, port int) error { func StartExecutor(logger *zap.Logger, fissionNamespace string, functionNamespace string, envBuilderNamespace string, port int) error {
// setup a signal handler for SIGTERM // setup a signal handler for SIGTERM
fission.SetupStackTraceHandler() fission.SetupStackTraceHandler()
@@ -202,33 +216,34 @@ func StartExecutor(fissionNamespace string, functionNamespace string, envBuilder
err = fissionClient.WaitForCRDs() err = fissionClient.WaitForCRDs()
if err != nil { if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err) return errors.Wrap(err, "error waiting for CRDs")
} }
restClient := fissionClient.GetCrdClient() restClient := fissionClient.GetCrdClient()
if err != nil { if err != nil {
log.Printf("Failed to get kubernetes client: %v", err) return errors.Wrap(err, "failed to get kubernetes client")
return err
} }
fsCache := fscache.MakeFunctionServiceCache() fsCache := fscache.MakeFunctionServiceCache(logger)
poolID := strings.ToLower(uniuri.NewLen(8)) poolID := strings.ToLower(uniuri.NewLen(8))
reaper.CleanupOldExecutorObjects(kubernetesClient, poolID) reaper.CleanupOldExecutorObjects(logger, kubernetesClient, poolID)
go reaper.CleanupRoleBindings(kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30) go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
gpm := poolmgr.MakeGenericPoolManager( gpm := poolmgr.MakeGenericPoolManager(
logger,
fissionClient, kubernetesClient, fissionClient, kubernetesClient,
functionNamespace, poolID) functionNamespace, poolID)
ndm := newdeploy.MakeNewDeploy( ndm := newdeploy.MakeNewDeploy(
logger,
fissionClient, kubernetesClient, restClient, fissionClient, kubernetesClient, restClient,
functionNamespace, poolID) functionNamespace, poolID)
api := MakeExecutor(gpm, ndm, fissionClient, fsCache) api := MakeExecutor(logger, gpm, ndm, fissionClient, fsCache)
go api.Serve(port) go api.Serve(port)
go serveMetric() go serveMetric(logger)
return nil return nil
} }
executor/executor_test.go
+13 -3
View File
@@ -33,6 +33,7 @@ import (
"testing" "testing"
"time" "time"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/labels"
@@ -44,6 +45,12 @@ import (
"github.com/fission/fission/executor/client" "github.com/fission/fission/executor/client"
) )
func panicIf(err error) {
if err != nil {
log.Panicf("Error: %v", err)
}
}
// return the number of pods in the given namespace matching the given labels // return the number of pods in the given namespace matching the given labels
func countPods(kubeClient *kubernetes.Clientset, ns string, labelz map[string]string) int { func countPods(kubeClient *kubernetes.Clientset, ns string, labelz map[string]string) int {
pods, err := kubeClient.Pods(ns).List(metav1.ListOptions{ pods, err := kubeClient.Pods(ns).List(metav1.ListOptions{
@@ -134,8 +141,11 @@ func TestExecutor(t *testing.T) {
createTestNamespace(kubeClient, functionNs) createTestNamespace(kubeClient, functionNs)
defer kubeClient.Namespaces().Delete(functionNs, nil) defer kubeClient.Namespaces().Delete(functionNs, nil)
logger, err := zap.NewDevelopment()
panicIf(err)
// make sure CRD types exist on cluster // make sure CRD types exist on cluster
err = crd.EnsureFissionCRDs(apiExtClient) err = crd.EnsureFissionCRDs(logger, apiExtClient)
if err != nil { if err != nil {
log.Panicf("failed to ensure crds: %v", err) log.Panicf("failed to ensure crds: %v", err)
} }
@@ -165,13 +175,13 @@ func TestExecutor(t *testing.T) {
// create poolmgr // create poolmgr
port := 9999 port := 9999
err = StartExecutor(fissionNs, functionNs, port) err = StartExecutor(logger, fissionNs, functionNs, "fission-builder", port)
if err != nil { if err != nil {
log.Panicf("failed to start poolmgr: %v", err) log.Panicf("failed to start poolmgr: %v", err)
} }
// connect poolmgr client // connect poolmgr client
poolmgrClient := client.MakeClient(fmt.Sprintf("http://localhost:%v", port)) poolmgrClient := client.MakeClient(logger, fmt.Sprintf("http://localhost:%v", port))
// Wait for pool to be created (we don't actually need to do // Wait for pool to be created (we don't actually need to do
// this, since the API should do the right thing in any case). // this, since the API should do the right thing in any case).
executor/fscache/functionServiceCache.go
+16 -12
View File
@@ -17,9 +17,11 @@ limitations under the License.
package fscache package fscache
import ( import (
"log" "fmt"
"time" "time"
"github.com/pkg/errors"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
@@ -57,6 +59,7 @@ type (
} }
FunctionServiceCache struct { FunctionServiceCache struct {
logger *zap.Logger
byFunction *cache.Cache // function-key -> funcSvc : map[string]*funcSvc byFunction *cache.Cache // function-key -> funcSvc : map[string]*funcSvc
byAddress *cache.Cache // address -> function : map[string]metav1.ObjectMeta byAddress *cache.Cache // address -> function : map[string]metav1.ObjectMeta
byFunctionUID *cache.Cache // function uid -> function : map[string]metav1.ObjectMeta byFunctionUID *cache.Cache // function uid -> function : map[string]metav1.ObjectMeta
@@ -91,8 +94,9 @@ func IsNameExistError(err error) bool {
return false return false
} }
func MakeFunctionServiceCache() *FunctionServiceCache { func MakeFunctionServiceCache(logger *zap.Logger) *FunctionServiceCache {
fsc := &FunctionServiceCache{ fsc := &FunctionServiceCache{
logger: logger.Named("function_service_cache"),
byFunction: cache.MakeCache(0, 0), byFunction: cache.MakeCache(0, 0),
byAddress: cache.MakeCache(0, 0), byAddress: cache.MakeCache(0, 0),
byFunctionUID: cache.MakeCache(0, 0), byFunctionUID: cache.MakeCache(0, 0),
@@ -122,14 +126,16 @@ func (fsc *FunctionServiceCache) service() {
} }
resp.objects = funcObjects resp.objects = funcObjects
case LOG: case LOG:
fsc.logger.Info("dumping function service cache")
funcCopy := fsc.byFunction.Copy() funcCopy := fsc.byFunction.Copy()
log.Printf("Cache has %v entries", len(funcCopy)) info := []string{}
for key, fsvcI := range funcCopy { for key, fsvcI := range funcCopy {
fsvc := fsvcI.(*FuncSvc) fsvc := fsvcI.(*FuncSvc)
for _, kubeObj := range fsvc.KubernetesObjects { for _, kubeObj := range fsvc.KubernetesObjects {
log.Printf("%v\t%v\t%v", key, kubeObj.Kind, kubeObj.Name) info = append(info, fmt.Sprintf("%v\t%v\t%v", key, kubeObj.Kind, kubeObj.Name))
} }
} }
fsc.logger.Info("function service cache", zap.Int("item_count", len(funcCopy)), zap.Strings("cache", info))
} }
req.responseChannel <- resp req.responseChannel <- resp
} }
@@ -196,9 +202,8 @@ func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {
if err != nil { if err != nil {
if IsNameExistError(err) { if IsNameExistError(err) {
err = nil err = nil
} } else {
if err != nil { err = errors.Wrap(err, "error caching fsvc")
log.Printf("error caching fsvc: %v", err)
} }
return nil, err return nil, err
} }
@@ -209,9 +214,8 @@ func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {
if err != nil { if err != nil {
if IsNameExistError(err) { if IsNameExistError(err) {
err = nil err = nil
} } else {
if err != nil { err = errors.Wrap(err, "error caching fsvc by function uid")
log.Printf("error caching fsvc by function uid: %v", err)
} }
return nil, err return nil, err
} }
@@ -278,12 +282,12 @@ func (fsc *FunctionServiceCache) ListOld(age time.Duration) ([]*FuncSvc, error)
} }
func (fsc *FunctionServiceCache) Log() { func (fsc *FunctionServiceCache) Log() {
log.Printf("--- FunctionService Cache Contents") fsc.logger.Info("--- FunctionService Cache Contents")
responseChannel := make(chan *fscResponse) responseChannel := make(chan *fscResponse)
fsc.requestChannel <- &fscRequest{ fsc.requestChannel <- &fscRequest{
requestType: LOG, requestType: LOG,
responseChannel: responseChannel, responseChannel: responseChannel,
} }
<-responseChannel <-responseChannel
log.Printf("--- FunctionService Cache Contents End") fsc.logger.Info("--- FunctionService Cache Contents End")
} }
executor/fscache/functionServiceCache_test.go
+11 -1
View File
@@ -5,6 +5,7 @@ import (
"testing" "testing"
"time" "time"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -12,8 +13,17 @@ import (
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
) )
func panicIf(err error) {
if err != nil {
log.Panicf("Error: %v", err)
}
}
func TestFunctionServiceCache(t *testing.T) { func TestFunctionServiceCache(t *testing.T) {
fsc := MakeFunctionServiceCache() logger, err := zap.NewDevelopment()
panicIf(err)
fsc := MakeFunctionServiceCache(logger)
if fsc == nil { if fsc == nil {
log.Panicf("error creating cache") log.Panicf("error creating cache")
} }
executor/newdeploy/newdeploy.go
+42 -15
View File
@@ -20,11 +20,11 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"log"
"path/filepath" "path/filepath"
"time" "time"
"github.com/hashicorp/go-multierror" "github.com/hashicorp/go-multierror"
"go.uber.org/zap"
asv1 "k8s.io/api/autoscaling/v1" asv1 "k8s.io/api/autoscaling/v1"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1" "k8s.io/api/extensions/v1beta1"
@@ -59,10 +59,9 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro
existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Get(deployName, metav1.GetOptions{}) existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Get(deployName, metav1.GetOptions{})
if err == nil { if err == nil {
if waitForDeploy { if waitForDeploy {
err = scaleDeployment(deploy.kubernetesClient, err = deploy.scaleDeployment(existingDepl.Namespace, existingDepl.Name, minScale)
existingDepl.Namespace, existingDepl.Name, minScale)
if err != nil { if err != nil {
log.Printf("Error scaling up deployment for function %v: %v", fn.Metadata.Name, err) deploy.logger.Error("error scaling up function deployment", zap.Error(err), zap.String("function", fn.Metadata.Name))
return nil, err return nil, err
} }
@@ -86,7 +85,11 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro
depl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Create(deployment) depl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Create(deployment)
if err != nil { if err != nil {
log.Printf("Error while creating deployment: %v", err) deploy.logger.Error("error while creating function deployment",
zap.Error(err),
zap.String("function", fn.Metadata.Name),
zap.String("deployment_name", deployName),
zap.String("deployment_namespace", deployNamespace))
return nil, err return nil, err
} }
@@ -105,25 +108,40 @@ func (deploy *NewDeploy) setupRBACObjs(deployNamespace string, fn *crd.Function)
// create fetcher SA in this ns, if not already created // create fetcher SA in this ns, if not already created
_, err := fission.SetupSA(deploy.kubernetesClient, fission.FissionFetcherSA, deployNamespace) _, err := fission.SetupSA(deploy.kubernetesClient, fission.FissionFetcherSA, deployNamespace)
if err != nil { if err != nil {
log.Printf("Error : %v creating %s in ns : %s for function: %s.%s", err, fission.FissionFetcherSA, deployNamespace, fn.Metadata.Name, fn.Metadata.Namespace) deploy.logger.Error("error creating fission fetcher service account for function",
zap.Error(err),
zap.String("service_account_name", fission.FissionFetcherSA),
zap.String("service_account_namespace", deployNamespace),
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
return err return err
} }
// create a cluster role binding for the fetcher SA, if not already created, granting access to do a get on packages in any ns // create a cluster role binding for the fetcher SA, if not already created, granting access to do a get on packages in any ns
err = fission.SetupRoleBinding(deploy.kubernetesClient, fission.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace) err = fission.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, fission.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace)
if err != nil { if err != nil {
log.Printf("Error : %v creating %s RoleBinding for function: %s.%s", err, fission.PackageGetterRB, fn.Metadata.Name, fn.Metadata.Namespace) deploy.logger.Error("error creating role binding for function",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
return err return err
} }
// create rolebinding in function namespace for fetcherSA.envNamespace to be able to get secrets and configmaps // create rolebinding in function namespace for fetcherSA.envNamespace to be able to get secrets and configmaps
err = fission.SetupRoleBinding(deploy.kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace) err = fission.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace)
if err != nil { if err != nil {
log.Printf("Error : %v creating %s RoleBinding for function %s.%s", err, fission.SecretConfigMapGetterRB, fn.Metadata.Name, fn.Metadata.Namespace) deploy.logger.Error("error creating role binding for function",
zap.Error(err),
zap.String("role_binding", fission.SecretConfigMapGetterRB),
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
return err return err
} }
log.Printf("Set up all RBAC objects for function : %s.%s", fn.Metadata.Name, fn.Metadata.Namespace) deploy.logger.Info("set up all RBAC objects for function",
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
return nil return nil
} }
@@ -188,7 +206,7 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen
fetcherResources, err := util.GetFetcherResources() fetcherResources, err := util.GetFetcherResources()
if err != nil { if err != nil {
log.Printf("Error while parsing fetcher resources: %v", err) deploy.logger.Error("error while parsing fetcher resources", zap.Error(err))
return nil, err return nil, err
} }
@@ -515,19 +533,28 @@ func (deploy *NewDeploy) cleanupNewdeploy(ns string, name string) error {
err := deploy.deleteSvc(ns, name) err := deploy.deleteSvc(ns, name)
if err != nil { if err != nil {
log.Printf("Error deleting service for newdeploy function %v in namespace %v, error: %v", name, ns, err) deploy.logger.Error("error deleting service for newdeploy function",
zap.Error(err),
zap.String("function_name", name),
zap.String("function_namespace", ns))
multierror.Append(multierr, err) multierror.Append(multierr, err)
} }
err = deploy.deleteHpa(ns, name) err = deploy.deleteHpa(ns, name)
if err != nil { if err != nil {
log.Printf("Error deleting HPA for newdeploy function %v in namespace %v, error: %v", name, ns, err) deploy.logger.Error("error deleting service for newdeploy function",
zap.Error(err),
zap.String("function_name", name),
zap.String("function_namespace", ns))
multierror.Append(multierr, err) multierror.Append(multierr, err)
} }
err = deploy.deleteDeployment(ns, name) err = deploy.deleteDeployment(ns, name)
if err != nil { if err != nil {
log.Printf("Error deleting deployment for newdeploy function %v in namespace %v, error: %v", name, ns, err) deploy.logger.Error("error deleting deployment for newdeploy function",
zap.Error(err),
zap.String("function_name", name),
zap.String("function_namespace", ns))
multierror.Append(multierr, err) multierror.Append(multierr, err)
} }
return multierr.ErrorOrNil() return multierr.ErrorOrNil()
executor/newdeploy/newdeploymgr.go
+63 -43
View File
@@ -19,7 +19,6 @@ package newdeploy
import ( import (
"context" "context"
"fmt" "fmt"
"log"
"os" "os"
"strconv" "strconv"
"strings" "strings"
@@ -29,6 +28,7 @@ import (
"github.com/fission/fission/throttler" "github.com/fission/fission/throttler"
multierror "github.com/hashicorp/go-multierror" multierror "github.com/hashicorp/go-multierror"
"github.com/pkg/errors" "github.com/pkg/errors"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1" "k8s.io/api/extensions/v1beta1"
k8sErrs "k8s.io/apimachinery/pkg/api/errors" k8sErrs "k8s.io/apimachinery/pkg/api/errors"
@@ -46,6 +46,8 @@ import (
type ( type (
NewDeploy struct { NewDeploy struct {
logger *zap.Logger
kubernetesClient *kubernetes.Clientset kubernetesClient *kubernetes.Clientset
fissionClient *crd.FissionClient fissionClient *crd.FissionClient
crdClient *rest.RESTClient crdClient *rest.RESTClient
@@ -72,6 +74,7 @@ type (
) )
func MakeNewDeploy( func MakeNewDeploy(
logger *zap.Logger,
fissionClient *crd.FissionClient, fissionClient *crd.FissionClient,
kubernetesClient *kubernetes.Clientset, kubernetesClient *kubernetes.Clientset,
crdClient *rest.RESTClient, crdClient *rest.RESTClient,
@@ -79,7 +82,7 @@ func MakeNewDeploy(
instanceID string, instanceID string,
) *NewDeploy { ) *NewDeploy {
log.Printf("Creating NewDeploy ExecutorType") logger.Info("creating NewDeploy ExecutorType")
fetcherImg := os.Getenv("FETCHER_IMAGE") fetcherImg := os.Getenv("FETCHER_IMAGE")
if len(fetcherImg) == 0 { if len(fetcherImg) == 0 {
@@ -92,19 +95,21 @@ func MakeNewDeploy(
if len(os.Getenv("ENABLE_ISTIO")) > 0 { if len(os.Getenv("ENABLE_ISTIO")) > 0 {
istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO")) istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
if err != nil { if err != nil {
log.Println("Failed to parse ENABLE_ISTIO") logger.Info("failed to parse 'ENABLE_ISTIO'")
} }
enableIstio = istio enableIstio = istio
} }
nd := &NewDeploy{ nd := &NewDeploy{
logger: logger.Named("new_deploy"),
fissionClient: fissionClient, fissionClient: fissionClient,
kubernetesClient: kubernetesClient, kubernetesClient: kubernetesClient,
crdClient: crdClient, crdClient: crdClient,
instanceID: instanceID, instanceID: instanceID,
namespace: namespace, namespace: namespace,
fsCache: fscache.MakeFunctionServiceCache(), fsCache: fscache.MakeFunctionServiceCache(logger),
throttler: throttler.MakeThrottler(1 * time.Minute), throttler: throttler.MakeThrottler(1 * time.Minute),
fetcherImg: fetcherImg, fetcherImg: fetcherImg,
@@ -142,14 +147,18 @@ func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controll
fn := obj.(*crd.Function) fn := obj.(*crd.Function)
_, err := deploy.createFunction(fn, true) _, err := deploy.createFunction(fn, true)
if err != nil { if err != nil {
log.Printf("Error eager creating function: %v", err) deploy.logger.Error("error eager creating function",
zap.Error(err),
zap.Any("function", fn))
} }
}, },
DeleteFunc: func(obj interface{}) { DeleteFunc: func(obj interface{}) {
fn := obj.(*crd.Function) fn := obj.(*crd.Function)
err := deploy.deleteFunction(fn) err := deploy.deleteFunction(fn)
if err != nil { if err != nil {
log.Printf("Error deleing the function: %v", err) deploy.logger.Error("error deleting function",
zap.Error(err),
zap.Any("function", fn))
} }
}, },
UpdateFunc: func(oldObj interface{}, newObj interface{}) { UpdateFunc: func(oldObj interface{}, newObj interface{}) {
@@ -157,7 +166,10 @@ func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controll
newFn := newObj.(*crd.Function) newFn := newObj.(*crd.Function)
err := deploy.updateFunction(oldFn, newFn) err := deploy.updateFunction(oldFn, newFn)
if err != nil { if err != nil {
log.Printf("Error deleing the function: %v", err) deploy.logger.Error("error updating function",
zap.Error(err),
zap.Any("old_function", oldFn),
zap.Any("new_function", newFn))
} }
}, },
}) })
@@ -186,7 +198,7 @@ func (deploy *NewDeploy) createFunction(fn *crd.Function, firstcreate bool) (*fs
fsvc, ok := fsvcObj.(*fscache.FuncSvc) fsvc, ok := fsvcObj.(*fscache.FuncSvc)
if !ok { if !ok {
log.Panic("Receive unknown object, expect pointer of function service object") deploy.logger.Panic("receive unknown object while creating function - expected pointer of function service object")
} }
return fsvc, err return fsvc, err
@@ -198,7 +210,7 @@ func (deploy *NewDeploy) deleteFunction(fn *crd.Function) error {
} }
err := deploy.fnDelete(fn) err := deploy.fnDelete(fn)
if err != nil { if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error deleting kubernetes objects of function %v", fn.Metadata)) err = errors.Wrapf(err, "error deleting kubernetes objects of function %v", fn.Metadata)
} }
return err return err
} }
@@ -235,22 +247,23 @@ func (deploy *NewDeploy) fnCreate(fn *crd.Function, firstcreate bool) (*fscache.
// then deployment) to take advantage of waiting time. // then deployment) to take advantage of waiting time.
svc, err := deploy.createOrGetSvc(deployLabels, objName, ns) svc, err := deploy.createOrGetSvc(deployLabels, objName, ns)
if err != nil { if err != nil {
log.Printf("Error creating the service %v: %v", objName, err) deploy.logger.Error("error creating service", zap.Error(err), zap.String("service", objName))
go deploy.cleanupNewdeploy(ns, objName) go deploy.cleanupNewdeploy(ns, objName)
return nil, errors.Wrap(err, fmt.Sprintf("error creating service %v", objName)) return nil, errors.Wrapf(err, "error creating service %v", objName)
} }
svcAddress := fmt.Sprintf("%v.%v", svc.Name, svc.Namespace) svcAddress := fmt.Sprintf("%v.%v", svc.Name, svc.Namespace)
depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels, ns, firstcreate) depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels, ns, firstcreate)
if err != nil { if err != nil {
log.Printf("Error creating the deployment %v: %v", objName, err) deploy.logger.Error("error creating deployment", zap.Error(err), zap.String("deployment", objName))
go deploy.cleanupNewdeploy(ns, objName) go deploy.cleanupNewdeploy(ns, objName)
return nil, errors.Wrap(err, fmt.Sprintf("error creating deployment %v", objName)) return nil, errors.Wrapf(err, "error creating deployment %v", objName)
} }
hpa, err := deploy.createOrGetHpa(objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl) hpa, err := deploy.createOrGetHpa(objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl)
if err != nil { if err != nil {
deploy.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName))
go deploy.cleanupNewdeploy(ns, objName) go deploy.cleanupNewdeploy(ns, objName)
return nil, errors.Wrap(err, fmt.Sprintf("error creating the HPA %v:", objName)) return nil, errors.Wrapf(err, "error creating the HPA %v", objName)
} }
kubeObjRefs := []apiv1.ObjectReference{ kubeObjRefs := []apiv1.ObjectReference{
@@ -292,7 +305,7 @@ func (deploy *NewDeploy) fnCreate(fn *crd.Function, firstcreate bool) (*fscache.
_, err = deploy.fsCache.Add(*fsvc) _, err = deploy.fsCache.Add(*fsvc)
if err != nil { if err != nil {
log.Printf("Error adding the function to cache: %v", err) deploy.logger.Error("error adding function to cache", zap.Error(err), zap.Any("function", fsvc.Function))
return fsvc, err return fsvc, err
} }
return fsvc, nil return fsvc, nil
@@ -313,7 +326,8 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
// Executor type is no longer New Deployment // Executor type is no longer New Deployment
if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy && if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy &&
oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy { oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy {
log.Printf("function does not use new deployment executor anymore, deleting resources: %v", newFn) deploy.logger.Info("function does not use new deployment executor anymore, deleting resources",
zap.Any("function", newFn))
// IMP - pass the oldFn, as the new/modified function is not in cache // IMP - pass the oldFn, as the new/modified function is not in cache
return deploy.deleteFunction(oldFn) return deploy.deleteFunction(oldFn)
} }
@@ -321,17 +335,19 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
// Executor type changed to New Deployment from something else // Executor type changed to New Deployment from something else
if oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy && if oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy &&
newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy { newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy {
log.Printf("function type changed to new deployment, creating resources: %v %v", oldFn.Metadata, newFn.Metadata) deploy.logger.Info("function type changed to new deployment, creating resources",
zap.Any("old_function", oldFn.Metadata),
zap.Any("new_function", newFn.Metadata))
_, err := deploy.createFunction(newFn, true) _, err := deploy.createFunction(newFn, true)
if err != nil { if err != nil {
updateStatus(oldFn, err, "error changing the function's type to newdeploy") deploy.updateStatus(oldFn, err, "error changing the function's type to newdeploy")
} }
return err return err
} }
fsvc, err := deploy.fsCache.GetByFunctionUID(newFn.Metadata.UID) fsvc, err := deploy.fsCache.GetByFunctionUID(newFn.Metadata.UID)
if err != nil { if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error updating function due to unable to find function service cache: %v", oldFn)) err = errors.Wrapf(err, "error updating function due to unable to find function service cache: %v", oldFn)
return err return err
} }
@@ -349,7 +365,7 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
hpa, err := deploy.getHpa(ns, fnObjName) hpa, err := deploy.getHpa(ns, fnObjName)
if err != nil { if err != nil {
updateStatus(oldFn, err, "error getting HPA while updating function") deploy.updateStatus(oldFn, err, "error getting HPA while updating function")
return err return err
} }
@@ -375,7 +391,7 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
if hpaChanged { if hpaChanged {
err := deploy.updateHpa(hpa) err := deploy.updateHpa(hpa)
if err != nil { if err != nil {
updateStatus(oldFn, err, "error updating HPA while updating function") deploy.updateStatus(oldFn, err, "error updating HPA while updating function")
return err return err
} }
} }
@@ -413,15 +429,15 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
env, err := deploy.fissionClient.Environments(newFn.Spec.Environment.Namespace). env, err := deploy.fissionClient.Environments(newFn.Spec.Environment.Namespace).
Get(newFn.Spec.Environment.Name) Get(newFn.Spec.Environment.Name)
if err != nil { if err != nil {
updateStatus(oldFn, err, "failed to get environment while updating function") deploy.updateStatus(oldFn, err, "failed to get environment while updating function")
return err return err
} }
deployLabels := deploy.getDeployLabels(oldFn, env) deployLabels := deploy.getDeployLabels(oldFn, env)
log.Printf("updating %v deployment due to function %v update", fnObjName, newFn.Metadata.Name) deploy.logger.Info("updating deployment due to function update", zap.String("deployment", fnObjName), zap.String("function", newFn.Metadata.Name))
newDeployment, err := deploy.getDeploymentSpec(newFn, env, fnObjName, deployLabels) newDeployment, err := deploy.getDeploymentSpec(newFn, env, fnObjName, deployLabels)
if err != nil { if err != nil {
updateStatus(oldFn, err, "failed to get new deployment spec while updating function") deploy.updateStatus(oldFn, err, "failed to get new deployment spec while updating function")
return err return err
} }
@@ -434,7 +450,7 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
err = deploy.updateDeployment(newDeployment, ns) err = deploy.updateDeployment(newDeployment, ns)
if err != nil { if err != nil {
updateStatus(oldFn, err, "failed to update deployment while updating function") deploy.updateStatus(oldFn, err, "failed to update deployment while updating function")
return err return err
} }
} }
@@ -506,13 +522,13 @@ func (deploy *NewDeploy) updateKubeObjRefRV(fsvc *fscache.FuncSvc, objKind strin
} }
} }
fsvc.KubernetesObjects = kubeObjs fsvc.KubernetesObjects = kubeObjs
return errors.New(fmt.Sprintf("error finding kubernetes object reference with kind: %v", objKind)) return fmt.Errorf("error finding kubernetes object reference with kind: %v", objKind)
} }
// updateStatus is a function which updates status of update. // updateStatus is a function which updates status of update.
// Current implementation only logs messages, in future it will update function status // Current implementation only logs messages, in future it will update function status
func updateStatus(fn *crd.Function, err error, message string) { func (deploy *NewDeploy) updateStatus(fn *crd.Function, err error, message string) {
log.Println(message, fn, err) deploy.logger.Info("function status update", zap.Error(err), zap.Any("function", fn), zap.String("message", message))
} }
// IsValid does a get on the service address to ensure it's a valid service, then // IsValid does a get on the service address to ensure it's a valid service, then
@@ -526,20 +542,20 @@ func (deploy *NewDeploy) IsValid(fsvc *fscache.FuncSvc) bool {
_, err := deploy.kubernetesClient.CoreV1().Services(service[1]).Get(service[0], metav1.GetOptions{}) _, err := deploy.kubernetesClient.CoreV1().Services(service[1]).Get(service[0], metav1.GetOptions{})
if err != nil { if err != nil {
log.Printf("Error validating service address for function %v: %v", fsvc.Function.Name, err) deploy.logger.Error("error validating function service address", zap.String("function", fsvc.Function.Name), zap.Error(err))
return false return false
} }
deployObj := getDeploymentObj(fsvc.KubernetesObjects) deployObj := getDeploymentObj(fsvc.KubernetesObjects)
if deployObj == nil { if deployObj == nil {
log.Printf("Deployment obj for function %v does not exist", fsvc.Function.Name) deploy.logger.Error("deployment obj for function does not exist", zap.String("function", fsvc.Function.Name))
return false return false
} }
currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1(). currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{}) Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
if err != nil { if err != nil {
log.Printf("Error validating deployment for function %v: %v", fsvc.Function.Name, err) deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
return false return false
} }
@@ -560,7 +576,7 @@ func (deploy *NewDeploy) idleObjectReaper() {
envs, err := deploy.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{}) envs, err := deploy.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil { if err != nil {
log.Fatalf("Failed to get environment list: %v", err) deploy.logger.Fatal("failed to get environment list", zap.Error(err))
} }
envList := make(map[types.UID]struct{}) envList := make(map[types.UID]struct{})
@@ -570,7 +586,7 @@ func (deploy *NewDeploy) idleObjectReaper() {
funcSvcs, err := deploy.fsCache.ListOld(deploy.idlePodReapTime) funcSvcs, err := deploy.fsCache.ListOld(deploy.idlePodReapTime)
if err != nil { if err != nil {
log.Printf("Error reaping idle pods: %v", err) deploy.logger.Error("error reaping idle pods", zap.Error(err))
continue continue
} }
@@ -582,8 +598,9 @@ func (deploy *NewDeploy) idleObjectReaper() {
// For function with the environment that no longer exists, executor // For function with the environment that no longer exists, executor
// scales down the deployment as usual and prints log to notify user. // scales down the deployment as usual and prints log to notify user.
if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok { if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
log.Printf("Environment %v for function %v no longer exists", deploy.logger.Error("function environment no longer exists",
fsvc.Environment.Metadata.Name, fsvc.Name) zap.String("environment", fsvc.Environment.Metadata.Name),
zap.String("function", fsvc.Name))
} }
fn, err := deploy.fissionClient.Functions(fsvc.Function.Namespace).Get(fsvc.Function.Name) fn, err := deploy.fissionClient.Functions(fsvc.Function.Namespace).Get(fsvc.Function.Name)
@@ -593,20 +610,20 @@ func (deploy *NewDeploy) idleObjectReaper() {
if k8sErrs.IsNotFound(err) && fsvc.Executor == fscache.NEWDEPLOY { if k8sErrs.IsNotFound(err) && fsvc.Executor == fscache.NEWDEPLOY {
continue continue
} }
log.Printf("Error getting function: %v", fsvc.Function.Name) deploy.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name))
continue continue
} }
deployObj := getDeploymentObj(fsvc.KubernetesObjects) deployObj := getDeploymentObj(fsvc.KubernetesObjects)
if deployObj == nil { if deployObj == nil {
log.Printf("Error finding deployment for function %v: %v", fsvc.Function.Name, err) deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
continue continue
} }
currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1(). currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{}) Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
if err != nil { if err != nil {
log.Printf("Error validating deployment for function %v: %v", fsvc.Function.Name, err) deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
continue continue
} }
@@ -617,9 +634,9 @@ func (deploy *NewDeploy) idleObjectReaper() {
continue continue
} }
err = scaleDeployment(deploy.kubernetesClient, deployObj.Namespace, deployObj.Name, minScale) err = deploy.scaleDeployment(deployObj.Namespace, deployObj.Name, minScale)
if err != nil { if err != nil {
log.Printf("Error scaling down deployment for function %v: %v", fsvc.Function.Name, err) deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
} }
} }
} }
@@ -635,9 +652,12 @@ func getDeploymentObj(kubeobjs []apiv1.ObjectReference) *apiv1.ObjectReference {
return nil return nil
} }
func scaleDeployment(client *kubernetes.Clientset, deplNS string, deplName string, replicas int32) error { func (deploy *NewDeploy) scaleDeployment(deplNS string, deplName string, replicas int32) error {
log.Printf("Scale deployment %v in namespace %v to replicas %v", deplName, deplNS, replicas) deploy.logger.Info("scaling deployment",
_, err := client.ExtensionsV1beta1().Deployments(deplNS).UpdateScale(deplName, &v1beta1.Scale{ zap.String("deployment", deplName),
zap.String("namespace", deplNS),
zap.Int32("replicas", replicas))
_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deplNS).UpdateScale(deplName, &v1beta1.Scale{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
Name: deplName, Name: deplName,
Namespace: deplNS, Namespace: deplNS,
executor/poolmgr/funcwatcher.go
+26 -9
View File
@@ -19,7 +19,7 @@ package poolmgr
import ( import (
"time" "time"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors" kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -72,11 +72,15 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
// setup rolebinding is tried, if it fails, we dont return. we just log an error and move on, because : // setup rolebinding is tried, if it fails, we dont return. we just log an error and move on, because :
// 1. not all functions have secrets and/or configmaps, so things will work without this rolebinding in that case. // 1. not all functions have secrets and/or configmaps, so things will work without this rolebinding in that case.
// 2. on the contrary, when the route is tried, the env fetcher logs will show a 403 forbidden message and same will be relayed to executor. // 2. on the contrary, when the route is tried, the env fetcher logs will show a 403 forbidden message and same will be relayed to executor.
err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs) err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
if err != nil { if err != nil {
log.Printf("Error : %v creating %s RoleBinding", err, fission.SecretConfigMapGetterRB) gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", fission.SecretConfigMapGetterRB))
} else { } else {
log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in func's ns for func : %s/%s", fission.FissionFetcherSA, envNs, fn.Metadata.Namespace, fn.Metadata.Name) gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namepsace", envNs),
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
} }
if istioEnabled { if istioEnabled {
@@ -126,7 +130,11 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
// create function istio service if it does not exist // create function istio service if it does not exist
_, err = kubernetesClient.CoreV1().Services(envNs).Create(&svc) _, err = kubernetesClient.CoreV1().Services(envNs).Create(&svc)
if err != nil && !kerrors.IsAlreadyExists(err) { if err != nil && !kerrors.IsAlreadyExists(err) {
log.Printf("Error creating function istio service: %v", err) gpm.logger.Error("error creating istio service for function",
zap.Error(err),
zap.String("service_name", svcName),
zap.String("function_name", fn.Metadata.Name),
zap.Any("selectors", sel))
} }
} }
}, },
@@ -149,7 +157,11 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
// delete function istio service // delete function istio service
err := kubernetesClient.CoreV1().Services(envNs).Delete(svcName, nil) err := kubernetesClient.CoreV1().Services(envNs).Delete(svcName, nil)
if err != nil && !kerrors.IsNotFound(err) { if err != nil && !kerrors.IsNotFound(err) {
log.Printf("Error deleting function istio service: %v", err) gpm.logger.Error("error deleting istio service for function",
zap.Error(err),
zap.String("service_name", svcName),
zap.String("function_name", fn.Metadata.Name))
} }
} }
}, },
@@ -178,13 +190,18 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
envNs = newFunc.Spec.Environment.Namespace envNs = newFunc.Spec.Environment.Namespace
} }
err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB, err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.SecretConfigMapGetterRB,
newFunc.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, newFunc.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole,
fission.FissionFetcherSA, envNs) fission.FissionFetcherSA, envNs)
if err != nil { if err != nil {
log.Printf("Error : %v creating GetSecretConfigMapRoleBinding", err) gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", fission.SecretConfigMapGetterRB))
} else { } else {
log.Printf("Set up rolebinding for fetcher SA in func's env ns : %s, in func's ns : %s, for func : %s", newFunc.Spec.Environment.Namespace, newFunc.Metadata.Namespace, newFunc.Metadata.Name) gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namepsace", envNs),
zap.String("function_name", newFunc.Metadata.Name),
zap.String("function_namespace", newFunc.Metadata.Namespace))
} }
} }
}, },
executor/poolmgr/gp.go
+37 -30
View File
@@ -19,7 +19,6 @@ package poolmgr
import ( import (
"context" "context"
"fmt" "fmt"
"log"
"math/rand" "math/rand"
"net" "net"
"os" "os"
@@ -30,6 +29,7 @@ import (
"github.com/dchest/uniuri" "github.com/dchest/uniuri"
multierror "github.com/hashicorp/go-multierror" multierror "github.com/hashicorp/go-multierror"
"github.com/pkg/errors" "github.com/pkg/errors"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1" "k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,6 +46,7 @@ import (
type ( type (
GenericPool struct { GenericPool struct {
logger *zap.Logger
env *crd.Environment env *crd.Environment
replicas int32 // num idle pods replicas int32 // num idle pods
deployment *v1beta1.Deployment // kubernetes deployment deployment *v1beta1.Deployment // kubernetes deployment
@@ -83,6 +84,7 @@ type (
) )
func MakeGenericPool( func MakeGenericPool(
logger *zap.Logger,
fissionClient *crd.FissionClient, fissionClient *crd.FissionClient,
kubernetesClient *kubernetes.Clientset, kubernetesClient *kubernetes.Clientset,
env *crd.Environment, env *crd.Environment,
@@ -94,7 +96,9 @@ func MakeGenericPool(
enableIstio bool, enableIstio bool,
collectorEndpoint string) (*GenericPool, error) { collectorEndpoint string) (*GenericPool, error) {
log.Printf("Creating pool for environment %v", env.Metadata) gpLogger := logger.Named("generic_pool")
gpLogger.Info("creating pool", zap.Any("environment", env.Metadata))
fetcherImage := os.Getenv("FETCHER_IMAGE") fetcherImage := os.Getenv("FETCHER_IMAGE")
if len(fetcherImage) == 0 { if len(fetcherImage) == 0 {
@@ -104,6 +108,7 @@ func MakeGenericPool(
// TODO: in general we need to provide the user a way to configure pools. Initial // TODO: in general we need to provide the user a way to configure pools. Initial
// replicas, autoscaling params, various timeouts, etc. // replicas, autoscaling params, various timeouts, etc.
gp := &GenericPool{ gp := &GenericPool{
logger: gpLogger,
env: env, env: env,
replicas: initialReplicas, // TODO make this an env param instead? replicas: initialReplicas, // TODO make this an env param instead?
requestChannel: make(chan *choosePodRequest), requestChannel: make(chan *choosePodRequest),
@@ -128,13 +133,12 @@ func MakeGenericPool(
gp.runtimeImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY")) gp.runtimeImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY"))
gp.fetcherImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("FETCHER_IMAGE_PULL_POLICY")) gp.fetcherImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("FETCHER_IMAGE_PULL_POLICY"))
log.Printf("fetcher image: %v, pull policy: %v", gp.fetcherImage, gp.fetcherImagePullPolicy) gpLogger.Info("fetcher", zap.String("image", gp.fetcherImage), zap.Any("pull_policy", gp.fetcherImagePullPolicy))
// create fetcher SA in this ns, if not already created // create fetcher SA in this ns, if not already created
_, err := fission.SetupSA(gp.kubernetesClient, fission.FissionFetcherSA, gp.namespace) _, err := fission.SetupSA(gp.kubernetesClient, fission.FissionFetcherSA, gp.namespace)
if err != nil { if err != nil {
log.Printf("Error : %v creating fetcher SA in ns : %s", err, gp.namespace) return nil, errors.Wrapf(err, "error creating fetcher service account in namespace %q", gp.namespace)
return nil, err
} }
// Labels for generic deployment/RS/pods. // Labels for generic deployment/RS/pods.
@@ -145,7 +149,7 @@ func MakeGenericPool(
if err != nil { if err != nil {
return nil, err return nil, err
} }
log.Printf("[%v] Deployment created", env.Metadata) gpLogger.Info("deployment created", zap.Any("environment", env.Metadata))
go gp.choosePodService() go gp.choosePodService()
@@ -195,7 +199,7 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
for { for {
// Retries took too long, error out. // Retries took too long, error out.
if time.Since(startTime) > gp.podReadyTimeout { if time.Since(startTime) > gp.podReadyTimeout {
log.Printf("[%v] Erroring out, timed out", newLabels) gp.logger.Error("timed out waiting for pod", zap.Any("labels", newLabels), zap.Duration("timeout", gp.podReadyTimeout))
return nil, errors.New("timeout: waited too long to get a ready pod") return nil, errors.New("timeout: waited too long to get a ready pod")
} }
@@ -220,7 +224,10 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
// add it to the list of ready pods // add it to the list of ready pods
readyPods = append(readyPods, &pod) readyPods = append(readyPods, &pod)
} }
log.Printf("[%v] found %v ready pods of %v total", newLabels, len(readyPods), len(podList.Items)) gp.logger.Info("found ready pods",
zap.Any("labels", newLabels),
zap.Int("ready_count", len(readyPods)),
zap.Int("total", len(podList.Items)))
// If there are no ready pods, wait and retry. // If there are no ready pods, wait and retry.
if len(readyPods) == 0 { if len(readyPods) == 0 {
@@ -241,14 +248,14 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
// modified, this should fail; in that case just // modified, this should fail; in that case just
// retry. // retry.
chosenPod.ObjectMeta.Labels = newLabels chosenPod.ObjectMeta.Labels = newLabels
log.Printf("relabeling pod: [%v]", chosenPod.ObjectMeta.Name) gp.logger.Info("relabeling pod", zap.String("pod", chosenPod.ObjectMeta.Name))
_, err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Update(chosenPod) _, err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Update(chosenPod)
if err != nil { if err != nil {
log.Printf("failed to relabel pod [%v]: %v", chosenPod.ObjectMeta.Name, err) gp.logger.Error("failed to relabel pod", zap.Error(err), zap.String("pod", chosenPod.ObjectMeta.Name))
continue continue
} }
} }
log.Printf("Chosen pod: %v (in %v)", chosenPod.ObjectMeta.Name, time.Since(startTime)) gp.logger.Info("chose pod", zap.String("pod", chosenPod.ObjectMeta.Name), zap.Duration("elapsed_time", time.Since(startTime)))
return chosenPod, nil return chosenPod, nil
} }
} }
@@ -267,7 +274,7 @@ func (gp *GenericPool) scheduleDeletePod(name string) {
// The sleep allows debugging or collecting logs from the pod before it's // The sleep allows debugging or collecting logs from the pod before it's
// cleaned up. (We need a better solutions for both those things; log // cleaned up. (We need a better solutions for both those things; log
// aggregation and storage will help.) // aggregation and storage will help.)
log.Printf("Error in pod '%v', scheduling cleanup", name) gp.logger.Error("error in pod - scheduling cleanup", zap.String("pod", name))
// Ignore sleep here if istio feature is enabled, function pod // Ignore sleep here if istio feature is enabled, function pod
// will be deleted after 6 mins (terminationGracePeriodSeconds). // will be deleted after 6 mins (terminationGracePeriodSeconds).
if !gp.useIstio { if !gp.useIstio {
@@ -319,7 +326,7 @@ func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metada
// tell fetcher to get the function. // tell fetcher to get the function.
fetcherUrl := gp.getSpecializeUrl(podIP) fetcherUrl := gp.getSpecializeUrl(podIP)
log.Printf("[%v] calling fetcher to copy function with fetcher url: %v", metadata.Name, fetcherUrl) gp.logger.Info("calling fetcher to copy function", zap.String("function", metadata.Name), zap.String("url", fetcherUrl))
fn, err := gp.fissionClient. fn, err := gp.fissionClient.
Functions(metadata.Namespace). Functions(metadata.Namespace).
@@ -356,9 +363,9 @@ func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metada
}, },
} }
log.Printf("[%v] specializing pod", metadata.Name) gp.logger.Info("specializing pod", zap.String("function", metadata.Name))
err = fetcherClient.MakeClient(fetcherUrl).Specialize(ctx, &specializeReq) err = fetcherClient.MakeClient(gp.logger, fetcherUrl).Specialize(ctx, &specializeReq)
if err != nil { if err != nil {
return err return err
} }
@@ -550,7 +557,7 @@ func (gp *GenericPool) createPool() error {
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment) depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment)
if err != nil { if err != nil {
log.Printf("Error creating deployment for %s in kubernetes, err: %v", deployment.Name, err) gp.logger.Error("error creating deployment in kubernetes", zap.Error(err), zap.String("deployment", deployment.Name))
return err return err
} }
gp.deployment = depl gp.deployment = depl
@@ -564,11 +571,9 @@ func (gp *GenericPool) waitForReadyPod() error {
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get( depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get(
gp.deployment.ObjectMeta.Name, metav1.GetOptions{}) gp.deployment.ObjectMeta.Name, metav1.GetOptions{})
if err != nil { if err != nil {
err = errors.Wrap(err, fmt.Sprintf( e := "error waiting for ready pod for deployment"
"Error waiting for ready pod of deployment %v in namespace %v", gp.logger.Error(e, zap.String("deployment", gp.deployment.ObjectMeta.Name), zap.String("namespace", gp.namespace))
gp.deployment.ObjectMeta.Name, gp.namespace)) return fmt.Errorf("%s %q in namespace %q", e, gp.deployment.ObjectMeta.Name, gp.namespace)
log.Print(err)
return err
} }
gp.deployment = depl gp.deployment = depl
@@ -582,7 +587,7 @@ func (gp *GenericPool) waitForReadyPod() error {
gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(), gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
}) })
if err != nil { if err != nil {
log.Printf("Error getting pod list after timeout waiting for ready pod: %v", err) gp.logger.Error("error getting pod list after timeout waiting for ready pod", zap.Error(err))
} }
// Since even single pod is not ready, choosing the first pod to inspect is a good approximation. In future this can be done better // Since even single pod is not ready, choosing the first pod to inspect is a good approximation. In future this can be done better
@@ -593,9 +598,8 @@ func (gp *GenericPool) waitForReadyPod() error {
multierr = multierror.Append(multierr, errors.New(fmt.Sprintf("%v: %v", cStatus.State.Waiting.Reason, cStatus.State.Waiting.Message))) multierr = multierror.Append(multierr, errors.New(fmt.Sprintf("%v: %v", cStatus.State.Waiting.Reason, cStatus.State.Waiting.Message)))
} }
} }
return errors.Errorf( return errors.Wrapf(multierr, "Timeout: waited too long for pod of deployment %v in namespace %v to be ready",
"Timeout: waited too long for pod of deployment %v in namespace %v to be ready, error: %v", gp.deployment.ObjectMeta.Name, gp.namespace)
gp.deployment.ObjectMeta.Name, gp.namespace, multierr)
} }
time.Sleep(1000 * time.Millisecond) time.Sleep(1000 * time.Millisecond)
} }
@@ -623,7 +627,7 @@ func (gp *GenericPool) createSvc(name string, labels map[string]string) (*apiv1.
} }
func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*fscache.FuncSvc, error) { func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
log.Printf("[%v] Choosing pod from pool", m.Name) gp.logger.Info("choosing pod from pool", zap.String("function", m.Name))
newLabels := gp.labelsForFunction(m) newLabels := gp.labelsForFunction(m)
if gp.useIstio { if gp.useIstio {
@@ -677,7 +681,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
gp.scheduleDeletePod(pod.ObjectMeta.Name) gp.scheduleDeletePod(pod.ObjectMeta.Name)
return nil, err return nil, err
} }
log.Printf("Specialized pod: %v", pod.ObjectMeta.Name) gp.logger.Info("specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
var svcHost string var svcHost string
if gp.useSvc && !gp.useIstio { if gp.useSvc && !gp.useIstio {
@@ -694,7 +698,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
} }
if svc.ObjectMeta.Name != svcName { if svc.ObjectMeta.Name != svcName {
gp.scheduleDeletePod(pod.ObjectMeta.Name) gp.scheduleDeletePod(pod.ObjectMeta.Name)
return nil, errors.New(fmt.Sprintf("sanity check failed for svc %v", svc.ObjectMeta.Name)) return nil, errors.Errorf("sanity check failed for svc %v", svc.ObjectMeta.Name)
} }
// the fission router isn't in the same namespace, so return a // the fission router isn't in the same namespace, so return a
@@ -704,7 +708,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
svc := fission.GetFunctionIstioServiceName(m.Name, m.Namespace) svc := fission.GetFunctionIstioServiceName(m.Name, m.Namespace)
svcHost = fmt.Sprintf("%v.%v:8888", svc, gp.namespace) svcHost = fmt.Sprintf("%v.%v:8888", svc, gp.namespace)
} else { } else {
log.Printf("Using pod IP for specialized pod") gp.logger.Info("using pod IP for specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP) svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP)
} }
@@ -746,7 +750,10 @@ func (gp *GenericPool) destroy() error {
err := gp.kubernetesClient.ExtensionsV1beta1(). err := gp.kubernetesClient.ExtensionsV1beta1().
Deployments(gp.namespace).Delete(gp.deployment.ObjectMeta.Name, &delOpt) Deployments(gp.namespace).Delete(gp.deployment.ObjectMeta.Name, &delOpt)
if err != nil { if err != nil {
log.Printf("Error destroying deployment: %v", err) gp.logger.Error("error destroying deployment",
zap.Error(err),
zap.String("deployment_name", gp.deployment.ObjectMeta.Name),
zap.String("deployment_namespace", gp.namespace))
return err return err
} }
return nil return nil
executor/poolmgr/gpm.go
+27 -18
View File
@@ -18,12 +18,12 @@ package poolmgr
import ( import (
"context" "context"
"log"
"os" "os"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
@@ -45,6 +45,8 @@ const (
type ( type (
GenericPoolManager struct { GenericPoolManager struct {
logger *zap.Logger
pools map[string]*GenericPool pools map[string]*GenericPool
kubernetesClient *kubernetes.Clientset kubernetesClient *kubernetes.Clientset
namespace string namespace string
@@ -78,18 +80,22 @@ type (
) )
func MakeGenericPoolManager( func MakeGenericPoolManager(
logger *zap.Logger,
fissionClient *crd.FissionClient, fissionClient *crd.FissionClient,
kubernetesClient *kubernetes.Clientset, kubernetesClient *kubernetes.Clientset,
functionNamespace string, functionNamespace string,
instanceId string) *GenericPoolManager { instanceId string) *GenericPoolManager {
gpmLogger := logger.Named("generic_pool_manager")
gpm := &GenericPoolManager{ gpm := &GenericPoolManager{
logger: gpmLogger,
pools: make(map[string]*GenericPool), pools: make(map[string]*GenericPool),
kubernetesClient: kubernetesClient, kubernetesClient: kubernetesClient,
namespace: functionNamespace, namespace: functionNamespace,
fissionClient: fissionClient, fissionClient: fissionClient,
functionEnv: cache.MakeCache(10*time.Second, 0), functionEnv: cache.MakeCache(10*time.Second, 0),
fsCache: fscache.MakeFunctionServiceCache(), fsCache: fscache.MakeFunctionServiceCache(gpmLogger),
instanceId: instanceId, instanceId: instanceId,
requestChannel: make(chan *request), requestChannel: make(chan *request),
idlePodReapTime: 2 * time.Minute, idlePodReapTime: 2 * time.Minute,
@@ -101,7 +107,7 @@ func MakeGenericPoolManager(
if len(os.Getenv("ENABLE_ISTIO")) > 0 { if len(os.Getenv("ENABLE_ISTIO")) > 0 {
istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO")) istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
if err != nil { if err != nil {
log.Println("Failed to parse ENABLE_ISTIO") gpmLogger.Info("failed to parse ENABLE_ISTIO")
} }
gpm.enableIstio = istio gpm.enableIstio = istio
} }
@@ -142,7 +148,7 @@ func (gpm *GenericPoolManager) service() {
ns = req.env.Metadata.Namespace ns = req.env.Metadata.Namespace
} }
pool, err = MakeGenericPool( pool, err = MakeGenericPool(gpm.logger,
gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize, gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize,
ns, gpm.namespace, gpm.fsCache, gpm.instanceId, gpm.enableIstio, ns, gpm.namespace, gpm.fsCache, gpm.instanceId, gpm.enableIstio,
gpm.collectorEndpoint) gpm.collectorEndpoint)
@@ -163,7 +169,7 @@ func (gpm *GenericPoolManager) service() {
if !ok || poolsize == 0 { if !ok || poolsize == 0 {
// Env no longer exists or pool size changed to zero // Env no longer exists or pool size changed to zero
log.Printf("Destroying generic pool for environment %v", pool.env.Metadata) gpm.logger.Info("eestroying generic pool", zap.Any("environment", pool.env.Metadata))
delete(gpm.pools, key) delete(gpm.pools, key)
// and delete the pool asynchronously. // and delete the pool asynchronously.
@@ -195,7 +201,7 @@ func (gpm *GenericPoolManager) CleanupPools(envs []crd.Environment) {
func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) { func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
// from Func -> get Env // from Func -> get Env
log.Printf("[%v] getting environment for function", metadata.Name) gpm.logger.Info("getting environment for function", zap.String("function", metadata.Name))
env, err := gpm.getFunctionEnv(metadata) env, err := gpm.getFunctionEnv(metadata)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -207,7 +213,7 @@ func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.
} }
// from GenericPool -> get one function container // from GenericPool -> get one function container
// (this also adds to the cache) // (this also adds to the cache)
log.Printf("[%v] getting function service from pool", metadata.Name) gpm.logger.Info("getting function service from pool", zap.String("function", metadata.Name))
return pool.GetFuncSvc(ctx, metadata) return pool.GetFuncSvc(ctx, metadata)
} }
@@ -228,7 +234,7 @@ func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Enviro
} }
// Get env from metadata // Get env from metadata
log.Printf("[%v] getting env", m) gpm.logger.Info("getting env", zap.Any("function", m))
env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name) env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -247,11 +253,11 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{}) envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil { if err != nil {
if fission.IsNetworkError(err) { if fission.IsNetworkError(err) {
log.Printf("Encountered network error, retrying: %v", err) gpm.logger.Error("encountered network error, retrying", zap.Error(err))
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
continue continue
} }
log.Fatalf("Failed to get environment list: %v", err) gpm.logger.Fatal("failed to get environment list", zap.Error(err))
} }
// Create pools for all envs. TODO: we should make this a bit less eager, only // Create pools for all envs. TODO: we should make this a bit less eager, only
@@ -264,7 +270,7 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
if gpm.getEnvPoolsize(&env) > 0 { if gpm.getEnvPoolsize(&env) > 0 {
_, err := gpm.GetPool(&envs.Items[i]) _, err := gpm.GetPool(&envs.Items[i])
if err != nil { if err != nil {
log.Printf("eager-create pool failed: %v", err) gpm.logger.Error("eager-create pool failed", zap.Error(err))
} }
} }
} }
@@ -292,7 +298,7 @@ func (gpm *GenericPoolManager) IsValid(fsvc *fscache.FuncSvc) bool {
if obj.Kind == "pod" { if obj.Kind == "pod" {
pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{}) pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{})
if err == nil && strings.Contains(fsvc.Address, pod.Status.PodIP) && fission.IsReadyPod(pod) { if err == nil && strings.Contains(fsvc.Address, pod.Status.PodIP) && fission.IsReadyPod(pod) {
log.Printf("Valid pod address : %s", fsvc.Address) gpm.logger.Info("valid pod address", zap.String("address", fsvc.Address))
return true return true
} }
} }
@@ -309,7 +315,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{}) envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil { if err != nil {
log.Fatalf("Failed to get environment list: %v", err) gpm.logger.Fatal("failed to get environment list", zap.Error(err))
} }
envList := make(map[types.UID]struct{}) envList := make(map[types.UID]struct{})
@@ -319,7 +325,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime) funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime)
if err != nil { if err != nil {
log.Printf("Error reaping idle pods: %v", err) gpm.logger.Error("error reaping idle pods", zap.Error(err))
continue continue
} }
@@ -331,8 +337,9 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
// For function with the environment that no longer exists, executor // For function with the environment that no longer exists, executor
// cleanups the idle pod as usual and prints log to notify user. // cleanups the idle pod as usual and prints log to notify user.
if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok { if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
log.Printf("Environment %v for function %v no longer exists", gpm.logger.Info("function environment no longer exists",
fsvc.Environment.Metadata.Name, fsvc.Name) zap.String("environment", fsvc.Environment.Metadata.Name),
zap.String("function", fsvc.Name))
} }
if fsvc.Environment.Spec.AllowedFunctionsPerContainer == fission.AllowedFunctionsPerContainerInfinite { if fsvc.Environment.Spec.AllowedFunctionsPerContainer == fission.AllowedFunctionsPerContainerInfinite {
@@ -341,7 +348,9 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime) deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime)
if err != nil { if err != nil {
log.Printf("Error deleting Kubernetes objects for fsvc '%v': %v", fsvc, err) gpm.logger.Error("error deleting Kubernetes objects for function service",
zap.Error(err),
zap.Any("service", fsvc))
} }
if !deleted { if !deleted {
@@ -349,7 +358,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
} }
for _, kubeobj := range fsvc.KubernetesObjects { for _, kubeobj := range fsvc.KubernetesObjects {
reaper.CleanupKubeObject(gpm.kubernetesClient, &kubeobj) reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &kubeobj)
} }
} }
} }
executor/poolmgr/packagewatcher.go
+27 -8
View File
@@ -19,7 +19,7 @@ package poolmgr
import ( import (
"time" "time"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/fields"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
@@ -39,7 +39,9 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien
k8sCache.ResourceEventHandlerFuncs{ k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) { AddFunc: func(obj interface{}) {
pkg := obj.(*crd.Package) pkg := obj.(*crd.Package)
log.Printf("List watch for package reported a new package addition: %s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace) gpm.logger.Info("list watch for package reported a new package addition",
zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namepsace", pkg.Metadata.Namespace))
// create or update role-binding for fetcher sa in env ns to be able to get the pkg contents from pkg namespace // create or update role-binding for fetcher sa in env ns to be able to get the pkg contents from pkg namespace
envNs := fissionfnNamespace envNs := fissionfnNamespace
@@ -49,13 +51,21 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien
// here, we return if we hit an error during rolebinding setup. this is because this rolebinding is mandatory for // here, we return if we hit an error during rolebinding setup. this is because this rolebinding is mandatory for
// every function's package to be loaded into its env. without that, there's no point to move forward. // every function's package to be loaded into its env. without that, there's no point to move forward.
err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs) err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
if err != nil { if err != nil {
log.Printf("Error creating %s for package: %s.%s, err: %v", fission.PackageGetterRB, pkg.Metadata.Name, pkg.Metadata.Namespace, err) gpm.logger.Error("error creating rolebinding for package",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namespace", pkg.Metadata.Namespace))
return return
} }
log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, pkg.Metadata.Namespace, pkg.Metadata.Name) gpm.logger.Info("successfully set up rolebinding for fetcher service account",
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namespace", envNs),
zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namespace", pkg.Metadata.Namespace))
}, },
UpdateFunc: func(oldObj, newObj interface{}) { UpdateFunc: func(oldObj, newObj interface{}) {
@@ -75,14 +85,23 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien
envNs = newPkg.Spec.Environment.Namespace envNs = newPkg.Spec.Environment.Namespace
} }
err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB, err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.PackageGetterRB,
newPkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, newPkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole,
fission.FissionFetcherSA, envNs) fission.FissionFetcherSA, envNs)
if err != nil { if err != nil {
log.Printf("Error : %v updating %s for package: %s.%s", err, fission.PackageGetterRB, newPkg.Metadata.Name, newPkg.Metadata.Namespace) gpm.logger.Error("error updating rolebinding for package",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("package_name", newPkg.Metadata.Name),
zap.String("package_namespace", newPkg.Metadata.Namespace))
return return
} }
log.Printf("Updated rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, newPkg.Metadata.Namespace, newPkg.Metadata.Name)
gpm.logger.Info("successfully updated rolebinding for fetcher service account",
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namespace", envNs),
zap.String("package_name", newPkg.Metadata.Name),
zap.String("package_namespace", newPkg.Metadata.Namespace))
} }
}, },
}) })
executor/reaper/reaper.go
+85 -47
View File
@@ -17,11 +17,10 @@ limitations under the License.
package reaper package reaper
import ( import (
"fmt"
"log"
"strings" "strings"
"time" "time"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
@@ -36,24 +35,24 @@ var (
) )
// CleanupOldExecutorObjects cleans up resources created by old executor instances // CleanupOldExecutorObjects cleans up resources created by old executor instances
func CleanupOldExecutorObjects(kubernetesClient *kubernetes.Clientset, instanceId string) { func CleanupOldExecutorObjects(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, instanceId string) {
go func() { go func() {
err := cleanup(kubernetesClient, instanceId) err := cleanup(logger, kubernetesClient, instanceId)
if err != nil { if err != nil {
// TODO retry reaper; logged and ignored for now // TODO retry reaper; logged and ignored for now
log.Printf("Failed to reaper: %v", err) logger.Error("Failed to cleanup old executor objects", zap.Error(err))
} }
}() }()
} }
func cleanup(client *kubernetes.Clientset, instanceId string) error { func cleanup(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
err := cleanupServices(client, instanceId) err := cleanupServices(logger, client, instanceId)
if err != nil { if err != nil {
return err return err
} }
err = cleanupHpa(client, instanceId) err = cleanupHpa(logger, client, instanceId)
if err != nil { if err != nil {
return err return err
} }
@@ -61,7 +60,7 @@ func cleanup(client *kubernetes.Clientset, instanceId string) error {
// Deployments are used for idle pools and can be cleaned up // Deployments are used for idle pools and can be cleaned up
// immediately. (We should "adopt" these instead of creating // immediately. (We should "adopt" these instead of creating
// a new pool.) // a new pool.)
err = cleanupDeployments(client, instanceId) err = cleanupDeployments(logger, client, instanceId)
if err != nil { if err != nil {
return err return err
} }
@@ -73,7 +72,7 @@ func cleanup(client *kubernetes.Clientset, instanceId string) error {
// time. // time.
time.Sleep(6 * time.Minute) time.Sleep(6 * time.Minute)
err = cleanupPods(client, instanceId) err = cleanupPods(logger, client, instanceId)
if err != nil { if err != nil {
return err return err
} }
@@ -82,31 +81,39 @@ func cleanup(client *kubernetes.Clientset, instanceId string) error {
} }
// CleanupKubeObject deletes given kubernetes object // CleanupKubeObject deletes given kubernetes object
func CleanupKubeObject(kubeClient *kubernetes.Clientset, kubeobj *apiv1.ObjectReference) { func CleanupKubeObject(logger *zap.Logger, kubeClient *kubernetes.Clientset, kubeobj *apiv1.ObjectReference) {
switch strings.ToLower(kubeobj.Kind) { switch strings.ToLower(kubeobj.Kind) {
case "pod": case "pod":
err := kubeClient.CoreV1().Pods(kubeobj.Namespace).Delete(kubeobj.Name, nil) err := kubeClient.CoreV1().Pods(kubeobj.Namespace).Delete(kubeobj.Name, nil)
logErr(fmt.Sprintf("cleaning up pod %v ", kubeobj.Name), err) if err != nil {
logger.Error("error cleaning up pod", zap.Error(err), zap.String("pod", kubeobj.Name))
}
case "service": case "service":
err := kubeClient.CoreV1().Services(kubeobj.Namespace).Delete(kubeobj.Name, nil) err := kubeClient.CoreV1().Services(kubeobj.Namespace).Delete(kubeobj.Name, nil)
logErr(fmt.Sprintf("cleaning up service %v ", kubeobj.Name), err) if err != nil {
logger.Error("error cleaning up service", zap.Error(err), zap.String("service", kubeobj.Name))
}
case "deployment": case "deployment":
err := kubeClient.ExtensionsV1beta1().Deployments(kubeobj.Namespace).Delete(kubeobj.Name, &delOpt) err := kubeClient.ExtensionsV1beta1().Deployments(kubeobj.Namespace).Delete(kubeobj.Name, &delOpt)
logErr(fmt.Sprintf("cleaning up deployment %v ", kubeobj.Name), err) if err != nil {
logger.Error("error cleaning up deployment", zap.Error(err), zap.String("deployment", kubeobj.Name))
}
case "horizontalpodautoscaler": case "horizontalpodautoscaler":
err := kubeClient.AutoscalingV1().HorizontalPodAutoscalers(kubeobj.Namespace).Delete(kubeobj.Name, nil) err := kubeClient.AutoscalingV1().HorizontalPodAutoscalers(kubeobj.Namespace).Delete(kubeobj.Name, nil)
logErr(fmt.Sprintf("cleaning up horizontalpodautoscaler %v ", kubeobj.Name), err) if err != nil {
logger.Error("error cleaning up horizontalpodautoscaler", zap.Error(err), zap.String("horizontalpodautoscaler", kubeobj.Name))
}
default: default:
log.Printf("There was an error identifying the object type: %v for obj: %v", kubeobj.Kind, kubeobj) logger.Error("Could not identifying the object type to clean up", zap.String("type", kubeobj.Kind), zap.Any("object", kubeobj))
} }
} }
func cleanupDeployments(client *kubernetes.Clientset, instanceId string) error { func cleanupDeployments(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
deploymentList, err := client.ExtensionsV1beta1().Deployments(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) deploymentList, err := client.ExtensionsV1beta1().Deployments(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil { if err != nil {
return err return err
@@ -114,24 +121,34 @@ func cleanupDeployments(client *kubernetes.Clientset, instanceId string) error {
for _, dep := range deploymentList.Items { for _, dep := range deploymentList.Items {
id, ok := dep.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL] id, ok := dep.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL]
if ok && id != instanceId { if ok && id != instanceId {
log.Printf("Cleaning up deployment %v", dep.ObjectMeta.Name) logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt) err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
logErr("cleaning up deployment", err) if err != nil {
logger.Error("error cleaning up deployment",
zap.Error(err),
zap.String("deployment_name", dep.ObjectMeta.Name),
zap.String("deployment_namespace", dep.ObjectMeta.Namespace))
}
// ignore err // ignore err
} }
// Backward compatibility with older label name // Backward compatibility with older label name
pid, pok := dep.ObjectMeta.Labels[fission.POOLMGR_INSTANCEID_LABEL] pid, pok := dep.ObjectMeta.Labels[fission.POOLMGR_INSTANCEID_LABEL]
if pok && pid != instanceId { if pok && pid != instanceId {
log.Printf("Cleaning up deployment %v", dep.ObjectMeta.Name) logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt) err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
logErr("cleaning up deployment", err) if err != nil {
logger.Error("error cleaning up deployment",
zap.Error(err),
zap.String("deployment_name", dep.ObjectMeta.Name),
zap.String("deployment_namespace", dep.ObjectMeta.Namespace))
}
// ignore err // ignore err
} }
} }
return nil return nil
} }
func cleanupPods(client *kubernetes.Clientset, instanceId string) error { func cleanupPods(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
podList, err := client.CoreV1().Pods(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) podList, err := client.CoreV1().Pods(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil { if err != nil {
return err return err
@@ -139,17 +156,27 @@ func cleanupPods(client *kubernetes.Clientset, instanceId string) error {
for _, pod := range podList.Items { for _, pod := range podList.Items {
id, ok := pod.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL] id, ok := pod.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL]
if ok && id != instanceId { if ok && id != instanceId {
log.Printf("Cleaning up pod %v", pod.ObjectMeta.Name) logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil) err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
logErr("cleaning up pod", err) if err != nil {
logger.Error("error cleaning up pod",
zap.Error(err),
zap.String("pod_name", pod.ObjectMeta.Name),
zap.String("pod_namespace", pod.ObjectMeta.Namespace))
}
// ignore err // ignore err
} }
// Backward compatibility with older label name // Backward compatibility with older label name
pid, pok := pod.ObjectMeta.Labels[fission.POOLMGR_INSTANCEID_LABEL] pid, pok := pod.ObjectMeta.Labels[fission.POOLMGR_INSTANCEID_LABEL]
if pok && pid != instanceId { if pok && pid != instanceId {
log.Printf("Cleaning up pod %v", pod.ObjectMeta.Name) logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil) err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
logErr("cleaning up pod", err) if err != nil {
logger.Error("error cleaning up pod",
zap.Error(err),
zap.String("pod_name", pod.ObjectMeta.Name),
zap.String("pod_namespace", pod.ObjectMeta.Namespace))
}
// ignore err // ignore err
} }
@@ -157,7 +184,7 @@ func cleanupPods(client *kubernetes.Clientset, instanceId string) error {
return nil return nil
} }
func cleanupServices(client *kubernetes.Clientset, instanceId string) error { func cleanupServices(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
svcList, err := client.CoreV1().Services(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) svcList, err := client.CoreV1().Services(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil { if err != nil {
return err return err
@@ -165,16 +192,21 @@ func cleanupServices(client *kubernetes.Clientset, instanceId string) error {
for _, svc := range svcList.Items { for _, svc := range svcList.Items {
id, ok := svc.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL] id, ok := svc.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL]
if ok && id != instanceId { if ok && id != instanceId {
log.Printf("Cleaning up svc %v", svc.ObjectMeta.Name) logger.Info("cleaning up service", zap.String("service", svc.ObjectMeta.Name))
err := client.CoreV1().Services(svc.ObjectMeta.Namespace).Delete(svc.ObjectMeta.Name, nil) err := client.CoreV1().Services(svc.ObjectMeta.Namespace).Delete(svc.ObjectMeta.Name, nil)
logErr("cleaning up service", err) if err != nil {
logger.Error("error cleaning up service",
zap.Error(err),
zap.String("service_name", svc.ObjectMeta.Name),
zap.String("service_namespace", svc.ObjectMeta.Namespace))
}
// ignore err // ignore err
} }
} }
return nil return nil
} }
func cleanupHpa(client *kubernetes.Clientset, instanceId string) error { func cleanupHpa(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
hpaList, err := client.AutoscalingV1().HorizontalPodAutoscalers(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) hpaList, err := client.AutoscalingV1().HorizontalPodAutoscalers(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil { if err != nil {
return err return err
@@ -183,9 +215,15 @@ func cleanupHpa(client *kubernetes.Clientset, instanceId string) error {
for _, hpa := range hpaList.Items { for _, hpa := range hpaList.Items {
id, ok := hpa.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL] id, ok := hpa.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL]
if ok && id != instanceId { if ok && id != instanceId {
log.Printf("Cleaning up HPA %v", hpa.ObjectMeta.Name) logger.Info("cleaning up HPA", zap.String("hpa", hpa.ObjectMeta.Name))
err := client.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Delete(hpa.ObjectMeta.Name, nil) err := client.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Delete(hpa.ObjectMeta.Name, nil)
logErr("cleaning up HPA", err) if err != nil {
logger.Error("error cleaning up HPA",
zap.Error(err),
zap.String("hpa_name", hpa.ObjectMeta.Name),
zap.String("hpa_namespace", hpa.ObjectMeta.Namespace))
}
// ignore err
} }
} }
@@ -193,22 +231,16 @@ func cleanupHpa(client *kubernetes.Clientset, instanceId string) error {
} }
func logErr(msg string, err error) {
if err != nil {
log.Printf("Error %v: %v", msg, err)
}
}
// CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or // CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or
// deletes the rolebindings completely if there are no Service Accounts in a rolebinding object. // deletes the rolebindings completely if there are no Service Accounts in a rolebinding object.
func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) { func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
for { for {
log.Println("Starting cleanupRoleBindings cycle") logger.Info("starting cleanupRoleBindings cycle")
// get all rolebindings ( just to be efficient, one call to kubernetes ) // get all rolebindings ( just to be efficient, one call to kubernetes )
rbList, err := client.RbacV1beta1().RoleBindings(meta_v1.NamespaceAll).List(meta_v1.ListOptions{}) rbList, err := client.RbacV1beta1().RoleBindings(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil { if err != nil {
// something wrong, but next iteration hopefully succeeds // something wrong, but next iteration hopefully succeeds
log.Printf("Error listing rolebindings in all ns, err: %v", err) logger.Error("error listing role bindings in all namespaces", zap.Error(err))
continue continue
} }
@@ -228,7 +260,7 @@ func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.Fissio
// we can list the functions once per role-binding. // we can list the functions once per role-binding.
funcList, err := fissionClient.Functions(roleBinding.Namespace).List(meta_v1.ListOptions{}) funcList, err := fissionClient.Functions(roleBinding.Namespace).List(meta_v1.ListOptions{})
if err != nil { if err != nil {
log.Printf("Error fetching environment list in : %s", roleBinding.Namespace) logger.Error("error fetching environment list in namespace", zap.Error(err), zap.String("namespace", roleBinding.Namespace))
continue continue
} }
@@ -275,7 +307,7 @@ func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.Fissio
// check if there is an env obj in saNs // check if there is an env obj in saNs
envList, err := fissionClient.Environments(saNs).List(meta_v1.ListOptions{}) envList, err := fissionClient.Environments(saNs).List(meta_v1.ListOptions{})
if err != nil { if err != nil {
log.Printf("Error fetching environment list in : %s", saNs) logger.Error("error fetching environment list in service account namespace", zap.Error(err), zap.String("namespace", saNs))
continue continue
} }
@@ -313,15 +345,21 @@ func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.Fissio
// finally, make a call to RemoveSAFromRoleBindingWithRetries for all the service accounts that need to be removed // finally, make a call to RemoveSAFromRoleBindingWithRetries for all the service accounts that need to be removed
// for the role-binding in this iteration // for the role-binding in this iteration
if len(saToRemove) != 0 { if len(saToRemove) != 0 {
log.Printf("saToRemove : %v for rolebinding : %s.%s", saToRemove, roleBinding.Name, roleBinding.Namespace) logger.Info("removing service accounts from role binding",
zap.Any("service_accounts", saToRemove),
zap.String("role_binding_name", roleBinding.Name),
zap.String("role_binding_namespace", roleBinding.Namespace))
// call this once in the end for each role-binding // call this once in the end for each role-binding
err = fission.RemoveSAFromRoleBindingWithRetries(client, roleBinding.Name, roleBinding.Namespace, saToRemove) err = fission.RemoveSAFromRoleBindingWithRetries(logger, client, roleBinding.Name, roleBinding.Namespace, saToRemove)
if err != nil { if err != nil {
// if there's an error, we just log it and proceed with the next role-binding, hoping that this role-binding // if there's an error, we just log it and proceed with the next role-binding, hoping that this role-binding
// will be processed in next iteration. // will be processed in next iteration.
log.Printf("Error removing SA : %v from rolebinding : %s.%s, err: %v", saToRemove, roleBinding.Name, logger.Info("error removing service account from role binding",
roleBinding.Namespace, err) zap.Error(err),
zap.Any("service_accounts", saToRemove),
zap.String("role_binding_name", roleBinding.Name),
zap.String("role_binding_namespace", roleBinding.Namespace))
} }
} }
} }
fission-bundle/main.go
+50 -44
View File
@@ -6,10 +6,10 @@ import (
"os" "os"
"strconv" "strconv"
docopt "github.com/docopt/docopt-go"
"go.opencensus.io/exporter/jaeger" "go.opencensus.io/exporter/jaeger"
"go.opencensus.io/trace" "go.opencensus.io/trace"
"go.uber.org/zap"
docopt "github.com/docopt/docopt-go"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/buildermgr" "github.com/fission/fission/buildermgr"
@@ -22,66 +22,66 @@ import (
"github.com/fission/fission/timer" "github.com/fission/fission/timer"
) )
func runController(port int) { func runController(logger *zap.Logger, port int) {
controller.Start(port, false) controller.Start(logger, port, false)
log.Fatalf("Error: Controller exited.") logger.Fatal("controller exited")
} }
func runRouter(port int, executorUrl string) { func runRouter(logger *zap.Logger, port int, executorUrl string) {
router.Start(port, executorUrl) router.Start(logger, port, executorUrl)
log.Fatalf("Error: Router exited.") logger.Fatal("router exited")
} }
func runExecutor(port int, fissionNamespace, functionNamespace, envBuilderNamespace string) { func runExecutor(logger *zap.Logger, port int, fissionNamespace, functionNamespace, envBuilderNamespace string) {
err := executor.StartExecutor(fissionNamespace, functionNamespace, envBuilderNamespace, port) err := executor.StartExecutor(logger, fissionNamespace, functionNamespace, envBuilderNamespace, port)
if err != nil { if err != nil {
log.Fatalf("Error starting executor: %v", err) logger.Fatal("error starting executor", zap.Error(err))
} }
} }
func runKubeWatcher(routerUrl string) { func runKubeWatcher(logger *zap.Logger, routerUrl string) {
err := kubewatcher.Start(routerUrl) err := kubewatcher.Start(logger, routerUrl)
if err != nil { if err != nil {
log.Fatalf("Error starting kubewatcher: %v", err) logger.Fatal("error starting kubewatcher", zap.Error(err))
} }
} }
func runTimer(routerUrl string) { func runTimer(logger *zap.Logger, routerUrl string) {
err := timer.Start(routerUrl) err := timer.Start(logger, routerUrl)
if err != nil { if err != nil {
log.Fatalf("Error starting timer: %v", err) logger.Fatal("error starting timer", zap.Error(err))
} }
} }
func runMessageQueueMgr(routerUrl string) { func runMessageQueueMgr(logger *zap.Logger, routerUrl string) {
err := messagequeue.Start(routerUrl) err := messagequeue.Start(logger, routerUrl)
if err != nil { if err != nil {
log.Fatalf("Error starting timer: %v", err) logger.Fatal("error starting message queue manager", zap.Error(err))
} }
} }
func runStorageSvc(port int, filePath string) { func runStorageSvc(logger *zap.Logger, port int, filePath string) {
subdir := os.Getenv("SUBDIR") subdir := os.Getenv("SUBDIR")
if len(subdir) == 0 { if len(subdir) == 0 {
subdir = "fission-functions" subdir = "fission-functions"
} }
enableArchivePruner := true enableArchivePruner := true
storagesvc.RunStorageService(storagesvc.StorageTypeLocal, storagesvc.RunStorageService(logger, storagesvc.StorageTypeLocal,
filePath, subdir, port, enableArchivePruner) filePath, subdir, port, enableArchivePruner)
} }
func runBuilderMgr(storageSvcUrl string, envBuilderNamespace string) { func runBuilderMgr(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) {
err := buildermgr.Start(storageSvcUrl, envBuilderNamespace) err := buildermgr.Start(logger, storageSvcUrl, envBuilderNamespace)
if err != nil { if err != nil {
log.Fatalf("Error starting buildermgr: %v", err) logger.Fatal("error starting builder manager", zap.Error(err))
} }
} }
func getPort(portArg interface{}) int { func getPort(logger *zap.Logger, portArg interface{}) int {
portArgStr := portArg.(string) portArgStr := portArg.(string)
port, err := strconv.Atoi(portArgStr) port, err := strconv.Atoi(portArgStr)
if err != nil { if err != nil {
log.Fatalf("Error: invalid port number '%v'", portArgStr) logger.Fatal("invalid port number", zap.Error(err), zap.String("port", portArgStr))
} }
return port return port
} }
@@ -94,10 +94,10 @@ func getStringArgWithDefault(arg interface{}, defaultValue string) string {
} }
} }
func registerTraceExporter(arguments map[string]interface{}) error { func registerTraceExporter(logger *zap.Logger, arguments map[string]interface{}) error {
collectorEndpoint := getStringArgWithDefault(arguments["--collectorEndpoint"], "") collectorEndpoint := getStringArgWithDefault(arguments["--collectorEndpoint"], "")
if collectorEndpoint == "" { if collectorEndpoint == "" {
log.Print("Skipping trace exporter registration") logger.Info("skipping trace exporter registration")
return nil return nil
} }
@@ -193,15 +193,21 @@ Options:
--builderMgr Start builder manager. --builderMgr Start builder manager.
--version Print version information --version Print version information
` `
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
version := fmt.Sprintf("Fission Bundle Version: %v", fission.BuildInfo().String()) version := fmt.Sprintf("Fission Bundle Version: %v", fission.BuildInfo().String())
arguments, err := docopt.Parse(usage, nil, true, version, false) arguments, err := docopt.Parse(usage, nil, true, version, false)
if err != nil { if err != nil {
log.Fatalf("Error: %v", err) logger.Fatal("Could not parse command line arguments", zap.Error(err))
} }
err = registerTraceExporter(arguments) err = registerTraceExporter(logger, arguments)
if err != nil { if err != nil {
log.Fatalf("Error: %v", err) logger.Fatal("Could not register trace exporter", zap.Error(err), zap.Any("argument", arguments))
} }
functionNs := getStringArgWithDefault(arguments["--namespace"], "fission-function") functionNs := getStringArgWithDefault(arguments["--namespace"], "fission-function")
@@ -213,40 +219,40 @@ Options:
storageSvcUrl := getStringArgWithDefault(arguments["--storageSvcUrl"], "http://storagesvc.fission") storageSvcUrl := getStringArgWithDefault(arguments["--storageSvcUrl"], "http://storagesvc.fission")
if arguments["--controllerPort"] != nil { if arguments["--controllerPort"] != nil {
port := getPort(arguments["--controllerPort"]) port := getPort(logger, arguments["--controllerPort"])
runController(port) runController(logger, port)
} }
if arguments["--routerPort"] != nil { if arguments["--routerPort"] != nil {
port := getPort(arguments["--routerPort"]) port := getPort(logger, arguments["--routerPort"])
runRouter(port, executorUrl) runRouter(logger, port, executorUrl)
} }
if arguments["--executorPort"] != nil { if arguments["--executorPort"] != nil {
port := getPort(arguments["--executorPort"]) port := getPort(logger, arguments["--executorPort"])
runExecutor(port, fissionNs, functionNs, envBuilderNs) runExecutor(logger, port, fissionNs, functionNs, envBuilderNs)
} }
if arguments["--kubewatcher"] == true { if arguments["--kubewatcher"] == true {
runKubeWatcher(routerUrl) runKubeWatcher(logger, routerUrl)
} }
if arguments["--timer"] == true { if arguments["--timer"] == true {
runTimer(routerUrl) runTimer(logger, routerUrl)
} }
if arguments["--mqt"] == true { if arguments["--mqt"] == true {
runMessageQueueMgr(routerUrl) runMessageQueueMgr(logger, routerUrl)
} }
if arguments["--builderMgr"] == true { if arguments["--builderMgr"] == true {
runBuilderMgr(storageSvcUrl, envBuilderNs) runBuilderMgr(logger, storageSvcUrl, envBuilderNs)
} }
if arguments["--storageServicePort"] != nil { if arguments["--storageServicePort"] != nil {
port := getPort(arguments["--storageServicePort"]) port := getPort(logger, arguments["--storageServicePort"])
filePath := arguments["--filePath"].(string) filePath := arguments["--filePath"].(string)
runStorageSvc(port, filePath) runStorageSvc(logger, port, filePath)
} }
select {} select {}
fission/logdb/influxdb.go
+1 -1
View File
@@ -19,7 +19,6 @@ package logdb
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"log"
"net/http" "net/http"
"net/url" "net/url"
"sort" "sort"
@@ -30,6 +29,7 @@ import (
influxdbClient "github.com/influxdata/influxdb/client/v2" influxdbClient "github.com/influxdata/influxdb/client/v2"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/fission/log"
) )
const ( const (
fission/logdb/logdb.go
+2 -4
View File
@@ -17,9 +17,8 @@ limitations under the License.
package logdb package logdb
import ( import (
"fmt"
"time" "time"
log "github.com/sirupsen/logrus"
) )
const ( const (
@@ -55,6 +54,5 @@ func GetLogDB(dbType string, serverURL string) (LogDatabase, error) {
case INFLUXDB: case INFLUXDB:
return NewInfluxDB(serverURL) return NewInfluxDB(serverURL)
} }
log.Fatalf("Log database type is incorrect, now only support %s", INFLUXDB) return nil, fmt.Errorf("log database type is incorrect, now only support %s", INFLUXDB)
return nil, nil
} }
fission/recorder.go
+1 -1
View File
@@ -18,7 +18,6 @@ package main
import ( import (
"fmt" "fmt"
"log"
"os" "os"
"strings" "strings"
"text/tabwriter" "text/tabwriter"
@@ -29,6 +28,7 @@ import (
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
"github.com/fission/fission/fission/log"
"github.com/fission/fission/fission/util" "github.com/fission/fission/fission/util"
) )
fission/records.go
+3 -3
View File
@@ -18,12 +18,12 @@ package main
import ( import (
"fmt" "fmt"
"log"
"os" "os"
"text/tabwriter" "text/tabwriter"
"github.com/urfave/cli" "github.com/urfave/cli"
"github.com/fission/fission/fission/log"
"github.com/fission/fission/fission/util" "github.com/fission/fission/fission/util"
"github.com/fission/fission/redis/build/gen" "github.com/fission/fission/redis/build/gen"
) )
@@ -31,7 +31,7 @@ import (
func recordsView(c *cli.Context) error { func recordsView(c *cli.Context) error {
var verbosity int var verbosity int
if c.Bool("v") && c.Bool("vv") { if c.Bool("v") && c.Bool("vv") {
log.Fatal("Conflicting verbosity levels, use either --v or --vv") log.Fatal("conflicting verbosity levels, use either --v or --vv")
} }
if c.Bool("v") { if c.Bool("v") {
verbosity = 1 verbosity = 1
@@ -47,7 +47,7 @@ func recordsView(c *cli.Context) error {
//Refuse multiple filters for now //Refuse multiple filters for now
if multipleFiltersSpecified(function, trigger, from+to) { if multipleFiltersSpecified(function, trigger, from+to) {
log.Fatal("Maximum of one filter is currently supported, either --function, --trigger, or --from,--to") log.Fatal("maximum of one filter is currently supported, either --function, --trigger, or --from,--to")
} }
if len(function) != 0 { if len(function) != 0 {
fission/replay.go
+1 -1
View File
@@ -18,10 +18,10 @@ package main
import ( import (
"fmt" "fmt"
"log"
"os" "os"
"text/tabwriter" "text/tabwriter"
"github.com/fission/fission/fission/log"
"github.com/fission/fission/fission/util" "github.com/fission/fission/fission/util"
"github.com/urfave/cli" "github.com/urfave/cli"
) )
fission/support/resources/crd.go
+10 -9
View File
@@ -17,13 +17,14 @@ limitations under the License.
package resources package resources
import ( import (
"log" "fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/controller/client" "github.com/fission/fission/controller/client"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
"github.com/fission/fission/fission/log"
) )
const ( const (
@@ -52,7 +53,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdEnvironment: case CrdEnvironment:
items, err := res.client.EnvironmentList(metav1.NamespaceAll) items, err := res.client.EnvironmentList(metav1.NamespaceAll)
if err != nil { if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err) log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return return
} }
@@ -64,7 +65,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdFunction: case CrdFunction:
items, err := res.client.FunctionList(metav1.NamespaceAll) items, err := res.client.FunctionList(metav1.NamespaceAll)
if err != nil { if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err) log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return return
} }
@@ -76,7 +77,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdPackage: case CrdPackage:
items, err := res.client.PackageList(metav1.NamespaceAll) items, err := res.client.PackageList(metav1.NamespaceAll)
if err != nil { if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err) log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return return
} }
@@ -89,7 +90,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdHttpTrigger: case CrdHttpTrigger:
items, err := res.client.HTTPTriggerList(metav1.NamespaceAll) items, err := res.client.HTTPTriggerList(metav1.NamespaceAll)
if err != nil { if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err) log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return return
} }
@@ -101,7 +102,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdKubeWatcher: case CrdKubeWatcher:
items, err := res.client.WatchList(metav1.NamespaceAll) items, err := res.client.WatchList(metav1.NamespaceAll)
if err != nil { if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err) log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return return
} }
@@ -116,7 +117,7 @@ func (res CrdDumper) Dump(dumpDir string) {
for _, mqType := range []string{fission.MessageQueueTypeNats, fission.MessageQueueTypeASQ} { for _, mqType := range []string{fission.MessageQueueTypeNats, fission.MessageQueueTypeASQ} {
l, err := res.client.MessageQueueTriggerList(mqType, metav1.NamespaceAll) l, err := res.client.MessageQueueTriggerList(mqType, metav1.NamespaceAll)
if err != nil { if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err) log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
break break
} }
triggers = append(triggers, l...) triggers = append(triggers, l...)
@@ -130,7 +131,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdTimeTrigger: case CrdTimeTrigger:
items, err := res.client.TimeTriggerList(metav1.NamespaceAll) items, err := res.client.TimeTriggerList(metav1.NamespaceAll)
if err != nil { if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err) log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return return
} }
@@ -140,7 +141,7 @@ func (res CrdDumper) Dump(dumpDir string) {
} }
default: default:
log.Printf("Unknown type: %v", res.crdType) log.Info(fmt.Sprintf("Unknown type: %v", res.crdType))
} }
} }
fission/support/resources/kubernetes.go
+13 -13
View File
@@ -21,7 +21,6 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"io" "io"
"log"
"path/filepath" "path/filepath"
"sync" "sync"
@@ -30,6 +29,7 @@ import (
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/fission/log"
) )
const ( const (
@@ -52,7 +52,7 @@ func NewKubernetesVersion(clientset *kubernetes.Clientset) Resource {
func (res KubernetesVersion) Dump(dumpDir string) { func (res KubernetesVersion) Dump(dumpDir string) {
serverVer, err := res.client.ServerVersion() serverVer, err := res.client.ServerVersion()
if err != nil { if err != nil {
log.Printf("Error setting up kubernetes client: %v", err) log.Info(fmt.Sprintf("Error setting up kubernetes client: %v", err))
return return
} }
@@ -80,7 +80,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesService: case KubernetesService:
objs, err := res.client.CoreV1().Services(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector}) objs, err := res.client.CoreV1().Services(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil { if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return return
} }
@@ -93,7 +93,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesDeployment: case KubernetesDeployment:
objs, err := res.client.AppsV1().Deployments(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector}) objs, err := res.client.AppsV1().Deployments(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil { if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return return
} }
@@ -105,7 +105,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesPod: case KubernetesPod:
objs, err := res.client.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector}) objs, err := res.client.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil { if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return return
} }
@@ -117,7 +117,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesHPA: case KubernetesHPA:
objs, err := res.client.AutoscalingV2beta1().HorizontalPodAutoscalers(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector}) objs, err := res.client.AutoscalingV2beta1().HorizontalPodAutoscalers(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil { if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return return
} }
@@ -129,7 +129,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesNode: case KubernetesNode:
objs, err := res.client.CoreV1().Nodes().List(metav1.ListOptions{LabelSelector: res.selector}) objs, err := res.client.CoreV1().Nodes().List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil { if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err) log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return return
} }
@@ -142,7 +142,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
} }
default: default:
log.Printf("Unknown type: %v", res.objType) log.Info(fmt.Sprintf("Unknown type: %v", res.objType))
} }
} }
@@ -187,7 +187,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
Pods(metav1.NamespaceAll). Pods(metav1.NamespaceAll).
List(metav1.ListOptions{LabelSelector: res.labelSelector}) List(metav1.ListOptions{LabelSelector: res.labelSelector})
if err != nil { if err != nil {
log.Printf("Error getting controller list: %v", err) log.Info(fmt.Sprintf("Error getting controller list: %v", err))
return return
} }
@@ -200,7 +200,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
defer wg.Done() defer wg.Done()
if !fission.IsReadyPod(&pod) { if !fission.IsReadyPod(&pod) {
fmt.Printf("Pod %v is not in ready state, ignore it\n", pod.Name) log.Info(fmt.Sprintf("Pod %v is not in ready state, ignore it\n", pod.Name))
return return
} }
@@ -211,7 +211,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
stream, err := req.Stream() stream, err := req.Stream()
if err != nil { if err != nil {
log.Printf("Error streaming logs for pod %v: %v", pod.Name, err) log.Info(fmt.Sprintf("Error streaming logs for pod %v: %v", pod.Name, err))
return return
} }
@@ -225,13 +225,13 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
stream.Close() stream.Close()
break break
} }
log.Printf("Error reading logs from buffer: %v", err) log.Info(fmt.Sprintf("Error reading logs from buffer: %v", err))
return return
} }
_, err = buffer.WriteString(string(line) + "\n") _, err = buffer.WriteString(string(line) + "\n")
if err != nil { if err != nil {
log.Printf("Error writing bytes to buffer: %v", err) log.Info(fmt.Sprintf("Error writing bytes to buffer: %v", err))
} }
} }
fission/support/resources/resource.go
+3 -3
View File
@@ -19,13 +19,13 @@ package resources
import ( import (
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log"
"path/filepath" "path/filepath"
"github.com/ghodss/yaml" "github.com/ghodss/yaml"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/fission/log"
) )
type Resource interface { type Resource interface {
@@ -40,7 +40,7 @@ func getFileName(dumpdir string, meta metav1.ObjectMeta) string {
func writeToFile(file string, obj interface{}) { func writeToFile(file string, obj interface{}) {
bs, err := yaml.Marshal(obj) bs, err := yaml.Marshal(obj)
if err != nil { if err != nil {
log.Printf("Error encoding object: %v", err) log.Info(fmt.Sprintf("Error encoding object: %v", err))
return return
} }
@@ -52,6 +52,6 @@ func writeToFile(file string, obj interface{}) {
err = ioutil.WriteFile(file, bs, 0644) err = ioutil.WriteFile(file, bs, 0644)
if err != nil { if err != nil {
log.Printf("Error writing file %v: %v", file, err) log.Info(fmt.Sprintf("Error writing file %v: %v", file, err))
} }
} }
fission/util/version.go
+2 -1
View File
@@ -3,11 +3,12 @@ package util
import ( import (
"fmt" "fmt"
yaml "gopkg.in/yaml.v2"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/controller/client" "github.com/fission/fission/controller/client"
"github.com/fission/fission/fission/log" "github.com/fission/fission/fission/log"
"github.com/fission/fission/fission/plugin" "github.com/fission/fission/fission/plugin"
yaml "gopkg.in/yaml.v2"
) )
// Versions is a container of versions of the client (and its plugins) and server (and its plugins). // Versions is a container of versions of the client (and its plugins) and server (and its plugins).
glide.lock
Generated
+21 -11
View File
@@ -1,5 +1,5 @@
hash: fbb8e5b2fc64114cf4cd7158a1498f36c580e01b327c14cd35682f999ce13db3 hash: 9c39373a78b1e542a42169b1af8fa644f62690e73a988a01bd5ad45a3639d872
updated: 2019-01-24T19:14:41.16237463+05:30 updated: 2019-03-04T16:38:40.987945-07:00
imports: imports:
- name: cloud.google.com/go - name: cloud.google.com/go
version: 3b1ae45394a234c385be014e9a488f2bb6eef821 version: 3b1ae45394a234c385be014e9a488f2bb6eef821
@@ -110,12 +110,10 @@ imports:
- openstack/identity/v3/tokens - openstack/identity/v3/tokens
- openstack/utils - openstack/utils
- pagination - pagination
- name: github.com/gorilla/context
version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
- name: github.com/gorilla/handlers - name: github.com/gorilla/handlers
version: 7e0847f9db758cdebd26c149d0ae9d5d0b9c98ce version: 7e0847f9db758cdebd26c149d0ae9d5d0b9c98ce
- name: github.com/gorilla/mux - name: github.com/gorilla/mux
version: e3702bed27f0d39777b0b37b664b6280e8ef8fbf version: a7962380ca08b5a188038c69871b8d3fbdf31e89
- name: github.com/graymeta/stow - name: github.com/graymeta/stow
version: 7b5498c561bbfeb17e413eb96c9edaf88af3855f version: 7b5498c561bbfeb17e413eb96c9edaf88af3855f
subpackages: subpackages:
@@ -162,7 +160,7 @@ imports:
subpackages: subpackages:
- pb - pb
- name: github.com/nats-io/nats-streaming-server - name: github.com/nats-io/nats-streaming-server
version: 7b758bb93407505bc1d0cf26d091fc281d99ca0c version: 247d7fa2c948ab918d7b4a9f761010082e8b92ea
subpackages: subpackages:
- spb - spb
- util - util
@@ -203,7 +201,7 @@ imports:
- name: github.com/satori/go.uuid - name: github.com/satori/go.uuid
version: f58768cc1a7a7e77a3bd49e98cdd21419399b6a3 version: f58768cc1a7a7e77a3bd49e98cdd21419399b6a3
- name: github.com/Shopify/sarama - name: github.com/Shopify/sarama
version: 03a43f93cd29dc549e6d9b11892795c206f9c38c version: 4602b5a8c6e826f9e0737865818dd43b2339a092
- name: github.com/sirupsen/logrus - name: github.com/sirupsen/logrus
version: 68cec9f21fbf3ea8d8f98c044bc6ce05f17b267a version: 68cec9f21fbf3ea8d8f98c044bc6ce05f17b267a
- name: github.com/spf13/pflag - name: github.com/spf13/pflag
@@ -223,7 +221,7 @@ imports:
- name: github.com/urfave/cli - name: github.com/urfave/cli
version: cfb38830724cc34fedffe9a2a29fb54fa9169cd1 version: cfb38830724cc34fedffe9a2a29fb54fa9169cd1
- name: go.opencensus.io - name: go.opencensus.io
version: 2b5032d79456124f42db6b7eb19ac6c155449dc2 version: f305e5c4e2cf345eba88de13d10de1126fa45a61
subpackages: subpackages:
- exemplar - exemplar
- exporter/jaeger - exporter/jaeger
@@ -240,6 +238,18 @@ imports:
- trace/internal - trace/internal
- trace/propagation - trace/propagation
- trace/tracestate - trace/tracestate
- name: go.uber.org/atomic
version: 1ea20fb1cbb1cc08cbd0d913a96dead89aa18289
- name: go.uber.org/multierr
version: 3c4937480c32f4c13a875a1829af76c98ca3d40a
- name: go.uber.org/zap
version: ff33455a0e382e8a81d14dd7c922020b6b5e7982
subpackages:
- buffer
- internal/bufferpool
- internal/color
- internal/exit
- zapcore
- name: golang.org/x/crypto - name: golang.org/x/crypto
version: 81e90905daefcd6fd217b62423c0908922eadb30 version: 81e90905daefcd6fd217b62423c0908922eadb30
subpackages: subpackages:
@@ -307,7 +317,7 @@ imports:
- name: gopkg.in/yaml.v2 - name: gopkg.in/yaml.v2
version: 670d4cfef0544295bc27a114dbac37980d83185a version: 670d4cfef0544295bc27a114dbac37980d83185a
- name: k8s.io/api - name: k8s.io/api
version: 0f11257a8a25954878633ebdc9841c67d8f83bdb version: c89978d5f86d7427bef2fc7752732c8c60b1d188
subpackages: subpackages:
- admissionregistration/v1alpha1 - admissionregistration/v1alpha1
- admissionregistration/v1beta1 - admissionregistration/v1beta1
@@ -338,7 +348,7 @@ imports:
- storage/v1alpha1 - storage/v1alpha1
- storage/v1beta1 - storage/v1beta1
- name: k8s.io/apiextensions-apiserver - name: k8s.io/apiextensions-apiserver
version: f584b16eb23bd2a3fd292a027d698d95db427c5d version: f0729a5940c5414a9b9da2f18ba8d5c4b559ec9a
subpackages: subpackages:
- pkg/apis/apiextensions - pkg/apis/apiextensions
- pkg/apis/apiextensions/v1beta1 - pkg/apis/apiextensions/v1beta1
@@ -346,7 +356,7 @@ imports:
- pkg/client/clientset/clientset/scheme - pkg/client/clientset/clientset/scheme
- pkg/client/clientset/clientset/typed/apiextensions/v1beta1 - pkg/client/clientset/clientset/typed/apiextensions/v1beta1
- name: k8s.io/apimachinery - name: k8s.io/apimachinery
version: e386b2658ed20923da8cc9250e552f082899a1ee version: d49e237a2683fa6dc43a86c7b1b766e0219fb6e7
subpackages: subpackages:
- pkg/api/errors - pkg/api/errors
- pkg/api/meta - pkg/api/meta
glide.yaml
+11 -3
View File
@@ -20,7 +20,9 @@ import:
version: ^1.1.0 version: ^1.1.0
- package: github.com/urfave/cli - package: github.com/urfave/cli
version: ^1.18.1 version: ^1.18.1
- package: golang.org/x/tools/imports - package: golang.org/x/tools
subpackages:
- imports
- package: golang.org/x/net - package: golang.org/x/net
subpackages: subpackages:
- context - context
@@ -68,12 +70,18 @@ import:
- package: github.com/prometheus/client_golang - package: github.com/prometheus/client_golang
version: v0.8.0 version: v0.8.0
- package: github.com/dustin/go-humanize - package: github.com/dustin/go-humanize
- package: github.com/golang/protobuf/proto - package: github.com/golang/protobuf
version: v1.1.0 version: v1.1.0
subpackages:
- proto
- package: github.com/bsm/sarama-cluster - package: github.com/bsm/sarama-cluster
version: ^2.1.11 version: ^2.1.11
- package: github.com/Shopify/sarama - package: github.com/Shopify/sarama
version: ^1.15.0 version: ^1.15.0
- package: go.opencensus.io - package: go.opencensus.io
version: ^0.19.0 version: ^0.19.0
- package: google.golang.org/api/support/bundler - package: google.golang.org/api
subpackages:
- support/bundler
- package: go.uber.org/zap
version: v1.9.1
kubewatcher/kubewatcher.go
+27 -21
View File
@@ -21,12 +21,12 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"log"
"reflect" "reflect"
"strings" "strings"
"sync/atomic" "sync/atomic"
"time" "time"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -48,6 +48,7 @@ const (
type ( type (
KubeWatcher struct { KubeWatcher struct {
logger *zap.Logger
watches map[types.UID]watchSubscription watches map[types.UID]watchSubscription
kubernetesClient *kubernetes.Clientset kubernetesClient *kubernetes.Clientset
requestChannel chan *kubeWatcherRequest requestChannel chan *kubeWatcherRequest
@@ -56,6 +57,7 @@ type (
} }
watchSubscription struct { watchSubscription struct {
logger *zap.Logger
watch crd.KubernetesWatchTrigger watch crd.KubernetesWatchTrigger
kubeWatch watch.Interface kubeWatch watch.Interface
lastResourceVersion string lastResourceVersion string
@@ -74,8 +76,9 @@ type (
} }
) )
func MakeKubeWatcher(kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher { func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher {
kw := &KubeWatcher{ kw := &KubeWatcher{
logger: logger.Named("kube_watcher"),
watches: make(map[types.UID]watchSubscription), watches: make(map[types.UID]watchSubscription),
kubernetesClient: kubernetesClient, kubernetesClient: kubernetesClient,
publisher: publisher, publisher: publisher,
@@ -167,16 +170,14 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *crd.KubernetesWa
case "JOB": case "JOB":
wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(listOptions) wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(listOptions)
default: default:
msg := fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type) err = errors.NewBadRequest(fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type))
log.Println(msg)
err = errors.NewBadRequest(msg)
} }
return wi, err return wi, err
} }
func (kw *KubeWatcher) addWatch(w *crd.KubernetesWatchTrigger) error { func (kw *KubeWatcher) addWatch(w *crd.KubernetesWatchTrigger) error {
log.Printf("Adding watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference) kw.logger.Info("adding watch", zap.String("name", w.Metadata.Name), zap.Any("function", w.Spec.FunctionReference))
ws, err := MakeWatchSubscription(w, kw.kubernetesClient, kw.publisher) ws, err := MakeWatchSubscription(kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher)
if err != nil { if err != nil {
return err return err
} }
@@ -185,7 +186,7 @@ func (kw *KubeWatcher) addWatch(w *crd.KubernetesWatchTrigger) error {
} }
func (kw *KubeWatcher) removeWatch(w *crd.KubernetesWatchTrigger) error { func (kw *KubeWatcher) removeWatch(w *crd.KubernetesWatchTrigger) error {
log.Printf("Removing watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference) kw.logger.Info("removing watch", zap.String("name", w.Metadata.Name), zap.Any("function", w.Spec.FunctionReference))
ws, ok := kw.watches[w.Metadata.UID] ws, ok := kw.watches[w.Metadata.UID]
if !ok { if !ok {
return fission.MakeError(fission.ErrorNotFound, return fission.MakeError(fission.ErrorNotFound,
@@ -211,9 +212,10 @@ func (kw *KubeWatcher) removeWatch(w *crd.KubernetesWatchTrigger) error {
// return nil // return nil
// } // }
func MakeWatchSubscription(w *crd.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) { func MakeWatchSubscription(logger *zap.Logger, w *crd.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) {
var stopped int32 = 0 var stopped int32 = 0
ws := &watchSubscription{ ws := &watchSubscription{
logger: logger.Named("watch_subscription"),
watch: *w, watch: *w,
kubeWatch: nil, kubeWatch: nil,
stopped: &stopped, stopped: &stopped,
@@ -234,8 +236,11 @@ func MakeWatchSubscription(w *crd.KubernetesWatchTrigger, kubeClient *kubernetes
func (ws *watchSubscription) restartWatch() error { func (ws *watchSubscription) restartWatch() error {
retries := 60 retries := 60
for { for {
log.Printf("(re)starting watch %v (ns:%v type:%v) at rv:%v", ws.logger.Info("(re)starting watch",
ws.watch.Metadata, ws.watch.Spec.Namespace, ws.watch.Spec.Type, ws.lastResourceVersion) zap.Any("watch", ws.watch.Metadata),
zap.String("namespace", ws.watch.Spec.Namespace),
zap.String("type", ws.watch.Spec.Type),
zap.String("last_resource_version", ws.lastResourceVersion))
wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion) wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion)
if err != nil { if err != nil {
retries-- retries--
@@ -260,7 +265,7 @@ func getResourceVersion(obj runtime.Object) (string, error) {
} }
func (ws *watchSubscription) eventDispatchLoop() { func (ws *watchSubscription) eventDispatchLoop() {
log.Println("Listening to watch ", ws.watch.Metadata.Name) ws.logger.Info("listening to watch", zap.String("name", ws.watch.Metadata.Name))
for { for {
// check watchSubscription is stopped or not before waiting for event // check watchSubscription is stopped or not before waiting for event
// comes from the kubeWatch.ResultChan(). This fix the edge case that // comes from the kubeWatch.ResultChan(). This fix the edge case that
@@ -273,14 +278,14 @@ func (ws *watchSubscription) eventDispatchLoop() {
if !more { if !more {
if ws.isStopped() { if ws.isStopped() {
// watch is removed by user. // watch is removed by user.
log.Println("Watch stopped", ws.watch.Metadata.Name) ws.logger.Info("watch stopped", zap.String("watch_name", ws.watch.Metadata.Name))
return return
} else { } else {
// watch closed due to timeout, restart it. // watch closed due to timeout, restart it.
log.Printf("Watch %v timed out, restarting", ws.watch.Metadata.Name) ws.logger.Info("watch timed out - restarting", zap.String("watch_name", ws.watch.Metadata.Name))
err := ws.restartWatch() err := ws.restartWatch()
if err != nil { if err != nil {
log.Panicf("Failed to restart watch: %v", err) ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name))
} }
continue continue
} }
@@ -288,19 +293,19 @@ func (ws *watchSubscription) eventDispatchLoop() {
if ev.Type == watch.Error { if ev.Type == watch.Error {
e := errors.FromObject(ev.Object) e := errors.FromObject(ev.Object)
log.Printf("Watch error, retrying in a second: %v", e) ws.logger.Info("watch error - retrying after one second", zap.Error(e), zap.String("watch_name", ws.watch.Metadata.Name))
// Start from the beginning to get around "too old resource version" // Start from the beginning to get around "too old resource version"
ws.lastResourceVersion = "" ws.lastResourceVersion = ""
time.Sleep(time.Second) time.Sleep(time.Second)
err := ws.restartWatch() err := ws.restartWatch()
if err != nil { if err != nil {
log.Panicf("Failed to restart watch: %v", err) ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name))
} }
continue continue
} }
rv, err := getResourceVersion(ev.Object) rv, err := getResourceVersion(ev.Object)
if err != nil { if err != nil {
log.Printf("Error getting resourceVersion from object: %v", err) ws.logger.Error("error getting resourceVersion from object", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name))
} else { } else {
ws.lastResourceVersion = rv ws.lastResourceVersion = rv
} }
@@ -309,7 +314,7 @@ func (ws *watchSubscription) eventDispatchLoop() {
var buf bytes.Buffer var buf bytes.Buffer
err = printKubernetesObject(ev.Object, &buf) err = printKubernetesObject(ev.Object, &buf)
if err != nil { if err != nil {
log.Printf("Failed to serialize object: %v", err) ws.logger.Error("failed to serialize object", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name))
// TODO send a POST request indicating error // TODO send a POST request indicating error
} }
@@ -322,8 +327,9 @@ func (ws *watchSubscription) eventDispatchLoop() {
// TODO support other function ref types. Or perhaps delegate to router? // TODO support other function ref types. Or perhaps delegate to router?
if ws.watch.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { if ws.watch.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName {
log.Printf("Error: unsupported function ref type: %v, can't publish event", ws.logger.Error("unsupported function ref type - cannot publish event",
ws.watch.Spec.FunctionReference.Type) zap.Any("type", ws.watch.Spec.FunctionReference.Type),
zap.String("watch_name", ws.watch.Metadata.Name))
continue continue
} }
kubewatcher/main.go
+8 -7
View File
@@ -17,26 +17,27 @@ limitations under the License.
package kubewatcher package kubewatcher
import ( import (
"log" "github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
"github.com/fission/fission/publisher" "github.com/fission/fission/publisher"
) )
func Start(routerUrl string) error { func Start(logger *zap.Logger, routerUrl string) error {
fissionClient, kubeClient, _, err := crd.MakeFissionClient() fissionClient, kubeClient, _, err := crd.MakeFissionClient()
if err != nil { if err != nil {
return err return errors.Wrap(err, "failed to get fission or kubernetes client")
} }
err = fissionClient.WaitForCRDs() err = fissionClient.WaitForCRDs()
if err != nil { if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err) return errors.Wrap(err, "error waiting for CRDs")
} }
poster := publisher.MakeWebhookPublisher(routerUrl) poster := publisher.MakeWebhookPublisher(logger, routerUrl)
kubeWatch := MakeKubeWatcher(kubeClient, poster) kubeWatch := MakeKubeWatcher(logger, kubeClient, poster)
MakeWatchSync(fissionClient, kubeWatch) MakeWatchSync(logger, fissionClient, kubeWatch)
return nil return nil
} }
kubewatcher/watchSync.go
+5 -3
View File
@@ -17,9 +17,9 @@ limitations under the License.
package kubewatcher package kubewatcher
import ( import (
"log"
"time" "time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
@@ -27,13 +27,15 @@ import (
type ( type (
WatchSync struct { WatchSync struct {
logger *zap.Logger
client *crd.FissionClient client *crd.FissionClient
kubeWatcher *KubeWatcher kubeWatcher *KubeWatcher
} }
) )
func MakeWatchSync(client *crd.FissionClient, kubeWatcher *KubeWatcher) *WatchSync { func MakeWatchSync(logger *zap.Logger, client *crd.FissionClient, kubeWatcher *KubeWatcher) *WatchSync {
ws := &WatchSync{ ws := &WatchSync{
logger: logger.Named("watch_sync"),
client: client, client: client,
kubeWatcher: kubeWatcher, kubeWatcher: kubeWatcher,
} }
@@ -46,7 +48,7 @@ func (ws *WatchSync) syncSvc() {
for { for {
watches, err := ws.client.KubernetesWatchTriggers(metav1.NamespaceAll).List(metav1.ListOptions{}) watches, err := ws.client.KubernetesWatchTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil { if err != nil {
log.Fatalf("Failed to get Kubernetes watch trigger list: %v", err) ws.logger.Fatal("failed to get Kubernetes watch trigger list", zap.Error(err))
} }
ws.kubeWatcher.Sync(watches.Items) ws.kubeWatcher.Sync(watches.Items)
mqtrigger/main.go
+7 -5
View File
@@ -17,22 +17,24 @@ limitations under the License.
package messagequeue package messagequeue
import ( import (
"log"
"os" "os"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
"github.com/fission/fission/mqtrigger/messageQueue" "github.com/fission/fission/mqtrigger/messageQueue"
) )
func Start(routerUrl string) error { func Start(logger *zap.Logger, routerUrl string) error {
fissionClient, _, _, err := crd.MakeFissionClient() fissionClient, _, _, err := crd.MakeFissionClient()
if err != nil { if err != nil {
log.Fatalf("Failed to get fission client: %v", err) return errors.Wrap(err, "failed to get fission or kubernetes client")
} }
err = fissionClient.WaitForCRDs() err = fissionClient.WaitForCRDs()
if err != nil { if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err) return errors.Wrap(err, "error waiting for CRDs")
} }
// Message queue type: nats is the only supported one for now // Message queue type: nats is the only supported one for now
@@ -42,6 +44,6 @@ func Start(routerUrl string) error {
MQType: mqType, MQType: mqType,
Url: mqUrl, Url: mqUrl,
} }
messageQueue.MakeMessageQueueTriggerManager(fissionClient, routerUrl, mqCfg) messageQueue.MakeMessageQueueTriggerManager(logger, fissionClient, routerUrl, mqCfg)
return nil return nil
} }
mqtrigger/messageQueue/asq.go
+44 -27
View File
@@ -19,7 +19,6 @@ package messageQueue
import ( import (
"bytes" "bytes"
"encoding/base64" "encoding/base64"
"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@@ -29,12 +28,12 @@ import (
"sync" "sync"
"time" "time"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
log "github.com/sirupsen/logrus"
"github.com/Azure/azure-sdk-for-go/storage"
) )
// TODO: some of these constants should probably be environment variables // TODO: some of these constants should probably be environment variables
@@ -55,6 +54,7 @@ const (
// AzureStorageConnection represents an Azure storage connection. // AzureStorageConnection represents an Azure storage connection.
type AzureStorageConnection struct { type AzureStorageConnection struct {
logger *zap.Logger
routerURL string routerURL string
service AzureQueueService service AzureQueueService
httpClient AzureHTTPClient httpClient AzureHTTPClient
@@ -173,7 +173,7 @@ func newAzureQueueService(client storage.Client) AzureQueueService {
} }
} }
func newAzureStorageConnection(routerURL string, config MessageQueueConfig) (MessageQueue, error) { func newAzureStorageConnection(logger *zap.Logger, routerURL string, config MessageQueueConfig) (MessageQueue, error) {
account := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME") account := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME")
if len(account) == 0 { if len(account) == 0 {
return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_NAME' is not set") return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_NAME' is not set")
@@ -184,13 +184,14 @@ func newAzureStorageConnection(routerURL string, config MessageQueueConfig) (Mes
return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_KEY' is not set") return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_KEY' is not set")
} }
log.Infof("Creating Azure storage connection to storage account '%s'.", account) logger.Info("creating Azure storage connection to storage account", zap.String("account", account))
client, err := storage.NewBasicClient(account, key) client, err := storage.NewBasicClient(account, key)
if err != nil { if err != nil {
return nil, fmt.Errorf("Failed to Azure create storage client: %v", err) return nil, errors.Wrap(err, "failed to create Azure storage client")
} }
return &AzureStorageConnection{ return &AzureStorageConnection{
logger: logger.Named("azue_storage"),
routerURL: routerURL, routerURL: routerURL,
service: newAzureQueueService(client), service: newAzureQueueService(client),
httpClient: &http.Client{ httpClient: &http.Client{
@@ -200,10 +201,10 @@ func newAzureStorageConnection(routerURL string, config MessageQueueConfig) (Mes
} }
func (asc AzureStorageConnection) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscription, error) { func (asc AzureStorageConnection) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscription, error) {
log.Infof("Subscribing to Azure storage queue '%s'.", trigger.Spec.Topic) asc.logger.Info("subscribing to Azure storage queue", zap.String("queue", trigger.Spec.Topic))
if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName {
return nil, fmt.Errorf("Unsupported function reference type (%v) for trigger %v", trigger.Spec.FunctionReference.Type, trigger.Metadata.Name) return nil, fmt.Errorf("unsupported function reference type (%v) for trigger %q", trigger.Spec.FunctionReference.Type, trigger.Metadata.Name)
} }
subscription := &AzureQueueSubscription{ subscription := &AzureQueueSubscription{
@@ -226,7 +227,7 @@ func (asc AzureStorageConnection) subscribe(trigger *crd.MessageQueueTrigger) (m
func (asc AzureStorageConnection) unsubscribe(subscription messageQueueSubscription) error { func (asc AzureStorageConnection) unsubscribe(subscription messageQueueSubscription) error {
sub := subscription.(*AzureQueueSubscription) sub := subscription.(*AzureQueueSubscription)
log.Infof("Unsubscribing from Azure storage queue '%s'.", sub.queueName) asc.logger.Info("unsubscribing from Azure storage queue", zap.String("queue", sub.queueName))
// Let the worker know we've unsubscribed // Let the worker know we've unsubscribed
sub.unsubscribe <- true sub.unsubscribe <- true
@@ -245,7 +246,7 @@ func runAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubsc
timer := time.NewTimer(AzureQueuePollingInterval) timer := time.NewTimer(AzureQueuePollingInterval)
for { for {
log.Infof("Waiting for %v before polling Azure storage queue '%s'.", AzureQueuePollingInterval, sub.queueName) conn.logger.Info("waiting before polling Azure storage queue", zap.Duration("interval_length", AzureQueuePollingInterval), zap.String("queue", sub.queueName))
select { select {
case <-sub.unsubscribe: case <-sub.unsubscribe:
timer.Stop() timer.Stop()
@@ -261,18 +262,18 @@ func runAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubsc
} }
func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubscription, wg *sync.WaitGroup) { func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubscription, wg *sync.WaitGroup) {
log.Infof("Polling messages for Azure storage queue '%s'.", sub.queueName) conn.logger.Info("polling for messages from Azure storage queue", zap.String("queue", sub.queueName))
err := sub.queue.Create(nil) err := sub.queue.Create(nil)
if err != nil { if err != nil {
log.Errorf("Failed to create message queue '%s': %v", sub.queueName, err) conn.logger.Error("failed to create message queue", zap.Error(err), zap.String("queue", sub.queueName))
return return
} }
for { for {
err := sub.queue.Create(nil) err := sub.queue.Create(nil)
if err != nil { if err != nil {
log.Errorf("Failed to create message queue '%s': %v", sub.queueName, err) conn.logger.Error("failed to create message queue", zap.Error(err), zap.String("queue", sub.queueName))
return return
} }
@@ -281,7 +282,7 @@ func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubs
VisibilityTimeout: int(AzureMessageVisibilityTimeout / time.Second), VisibilityTimeout: int(AzureMessageVisibilityTimeout / time.Second),
}) })
if err != nil { if err != nil {
log.Errorf("Failed to retrieve messages from Azure storage queue '%s': %v", sub.queueName, err) conn.logger.Error("failed to retrieve messages from Azure storage queue", zap.Error(err), zap.String("queue", sub.queueName))
break break
} }
if len(messages) == 0 { if len(messages) == 0 {
@@ -301,15 +302,15 @@ func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubs
func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscription, message AzureMessage) { func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscription, message AzureMessage) {
defer message.Delete(nil) defer message.Delete(nil)
log.Printf("Making HTTP request to %s.", sub.functionURL) conn.logger.Info("making HTTP request to invoke function", zap.String("function_url", sub.functionURL))
for i := 0; i <= AzureQueueRetryLimit; i++ { for i := 0; i <= AzureQueueRetryLimit; i++ {
if i > 0 { if i > 0 {
log.Infof("Retry #%d for request to %s.", i, sub.functionURL) conn.logger.Info("retrying function invocation", zap.Int("retry", i), zap.String("function_url", sub.functionURL))
} }
request, err := http.NewRequest("POST", sub.functionURL, bytes.NewReader(message.Bytes())) request, err := http.NewRequest("POST", sub.functionURL, bytes.NewReader(message.Bytes()))
if err != nil { if err != nil {
log.Errorf("Failed to create HTTP request to %s: %v", sub.functionURL, err) conn.logger.Error("failed to create HTTP request to invoke function", zap.Error(err), zap.String("function_url", sub.functionURL))
continue continue
} }
@@ -324,19 +325,22 @@ func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscri
response, err := conn.httpClient.Do(request) response, err := conn.httpClient.Do(request)
if err != nil { if err != nil {
log.Errorf("Request to %s failed: %v", sub.functionURL, err) conn.logger.Error("sending function invocation request failed", zap.Error(err), zap.String("function_url", sub.functionURL))
continue continue
} }
defer response.Body.Close() defer response.Body.Close()
body, err := ioutil.ReadAll(response.Body) body, err := ioutil.ReadAll(response.Body)
if err != nil { if err != nil {
log.Errorf("Failed to read response body from %s: %v.", sub.functionURL, err) conn.logger.Error("failed to read response body from function invocation", zap.Error(err), zap.String("function_url", sub.functionURL))
continue continue
} }
if response.StatusCode < 200 || response.StatusCode >= 300 { if response.StatusCode < 200 || response.StatusCode >= 300 {
log.Printf("Request to %s returned failure: %s (%d).", sub.functionURL, string(body), response.StatusCode) conn.logger.Error("function invocation request returned a failure status code",
zap.String("function_url", sub.functionURL),
zap.String("body", string(body)),
zap.Int("status_code", response.StatusCode))
continue continue
} }
@@ -344,14 +348,19 @@ func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscri
outputQueue := conn.service.GetQueue(sub.outputQueueName) outputQueue := conn.service.GetQueue(sub.outputQueueName)
err = outputQueue.Create(nil) err = outputQueue.Create(nil)
if err != nil { if err != nil {
log.Errorf("Failed to create output queue '%s': %v.", sub.outputQueueName, err) conn.logger.Error("failed to create output queue",
zap.Error(err),
zap.String("output_queue", sub.outputQueueName),
zap.String("function_url", sub.functionURL))
return return
} }
outputMessage := outputQueue.NewMessage(string(body)) outputMessage := outputQueue.NewMessage(string(body))
err = outputMessage.Put(nil) err = outputMessage.Put(nil)
if err != nil { if err != nil {
log.Errorf("Failed to post response body from %s to output queue '%s': %v.", sub.functionURL, sub.outputQueueName, err) conn.logger.Error("failed to post response body from function invocation to output queue",
zap.String("output_queue", sub.outputQueueName),
zap.String("function_url", sub.functionURL))
return return
} }
} }
@@ -360,20 +369,28 @@ func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscri
return return
} }
log.Errorf("Request to %s failed after %d retries; moving message to poison queue.", sub.functionURL, AzureQueueRetryLimit) conn.logger.Error("function invocation retired too many times - moving message to poison queue",
zap.Int("retry_limit", AzureQueueRetryLimit),
zap.String("function_url", sub.functionURL))
poisonQueueName := sub.queueName + AzurePoisonQueueSuffix poisonQueueName := sub.queueName + AzurePoisonQueueSuffix
poisonQueue := conn.service.GetQueue(poisonQueueName) poisonQueue := conn.service.GetQueue(poisonQueueName)
err := poisonQueue.Create(nil) err := poisonQueue.Create(nil)
if err != nil { if err != nil {
log.Errorf("Failed to create poison queue '%s': %v", poisonQueueName, err) conn.logger.Error("failed to create poison queue",
zap.Error(err),
zap.String("poison_queue_name", poisonQueueName),
zap.String("function_url", sub.functionURL))
return return
} }
poisonMessage := poisonQueue.NewMessage(string(message.Bytes())) poisonMessage := poisonQueue.NewMessage(string(message.Bytes()))
err = poisonMessage.Put(nil) err = poisonMessage.Put(nil)
if err != nil { if err != nil {
log.Errorf("Failed to post response body from %s to output queue '%s': %v", sub.functionURL, poisonQueueName, err) conn.logger.Error("failed to post response body from function invocation failure poison queue",
zap.Error(err),
zap.String("poison_queue_name", poisonQueueName),
zap.String("function_url", sub.functionURL))
return return
} }
} }
mqtrigger/messageQueue/asq_test.go
+32 -8
View File
@@ -18,26 +18,33 @@ package messageQueue
import ( import (
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"os" "os"
"strings" "strings"
"testing" "testing"
"time" "time"
"github.com/fission/fission"
"github.com/fission/fission/crd"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/Azure/azure-sdk-for-go/storage" "github.com/Azure/azure-sdk-for-go/storage"
"github.com/stretchr/testify/mock" "github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/crd"
) )
const ( const (
DummyRouterURL = "http://localhost" DummyRouterURL = "http://localhost"
) )
func panicIf(err error) {
if err != nil {
log.Panicf("Error: %v", err)
}
}
type azureQueueServiceMock struct { type azureQueueServiceMock struct {
mock.Mock mock.Mock
} }
@@ -103,7 +110,10 @@ func (m *azureHTTPClientMock) Do(req *http.Request) (*http.Response, error) {
} }
func TestNewStorageConnectionMissingAccountName(t *testing.T) { func TestNewStorageConnectionMissingAccountName(t *testing.T) {
connection, err := newAzureStorageConnection(DummyRouterURL, MessageQueueConfig{ logger, err := zap.NewDevelopment()
panicIf(err)
connection, err := newAzureStorageConnection(logger, DummyRouterURL, MessageQueueConfig{
MQType: fission.MessageQueueTypeASQ, MQType: fission.MessageQueueTypeASQ,
Url: "", Url: "",
}) })
@@ -112,8 +122,11 @@ func TestNewStorageConnectionMissingAccountName(t *testing.T) {
} }
func TestNewStorageConnectionMissingAccessKey(t *testing.T) { func TestNewStorageConnectionMissingAccessKey(t *testing.T) {
logger, err := zap.NewDevelopment()
panicIf(err)
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname") _ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname")
connection, err := newAzureStorageConnection(DummyRouterURL, MessageQueueConfig{ connection, err := newAzureStorageConnection(logger, DummyRouterURL, MessageQueueConfig{
MQType: fission.MessageQueueTypeASQ, MQType: fission.MessageQueueTypeASQ,
Url: "", Url: "",
}) })
@@ -123,9 +136,12 @@ func TestNewStorageConnectionMissingAccessKey(t *testing.T) {
} }
func TestNewStorageConnection(t *testing.T) { func TestNewStorageConnection(t *testing.T) {
logger, err := zap.NewDevelopment()
panicIf(err)
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname") _ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname")
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_KEY", "bm90IGEga2V5") _ = os.Setenv("AZURE_STORAGE_ACCOUNT_KEY", "bm90IGEga2V5")
connection, err := newAzureStorageConnection(DummyRouterURL, MessageQueueConfig{ connection, err := newAzureStorageConnection(logger, DummyRouterURL, MessageQueueConfig{
MQType: "azure-storage-queue", MQType: "azure-storage-queue",
Url: "", Url: "",
}) })
@@ -276,8 +292,12 @@ func TestAzureStorageQueuePoisonMessage(t *testing.T) {
service.On("GetQueue", QueueName).Return(queue).Once() service.On("GetQueue", QueueName).Return(queue).Once()
service.On("GetQueue", QueueName+AzurePoisonQueueSuffix).Return(poisonQueue).Once() service.On("GetQueue", QueueName+AzurePoisonQueueSuffix).Return(poisonQueue).Once()
logger, err := zap.NewDevelopment()
panicIf(err)
// Create the storage connection and subscribe to the trigger // Create the storage connection and subscribe to the trigger
connection := AzureStorageConnection{ connection := AzureStorageConnection{
logger: logger,
routerURL: DummyRouterURL, routerURL: DummyRouterURL,
service: service, service: service,
httpClient: httpClient, httpClient: httpClient,
@@ -420,8 +440,12 @@ func runAzureStorageQueueTest(t *testing.T, count int, output bool) {
service.On("GetQueue", OutputQueueName).Return(outputQueue).Times(count) service.On("GetQueue", OutputQueueName).Return(outputQueue).Times(count)
} }
logger, err := zap.NewDevelopment()
panicIf(err)
// Create the storage connection and subscribe to the trigger // Create the storage connection and subscribe to the trigger
connection := AzureStorageConnection{ connection := AzureStorageConnection{
logger: logger,
routerURL: DummyRouterURL, routerURL: DummyRouterURL,
service: service, service: service,
httpClient: httpClient, httpClient: httpClient,
mqtrigger/messageQueue/kafka.go
+51 -27
View File
@@ -26,7 +26,7 @@ import (
sarama "github.com/Shopify/sarama" sarama "github.com/Shopify/sarama"
cluster "github.com/bsm/sarama-cluster" cluster "github.com/bsm/sarama-cluster"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
@@ -34,30 +34,35 @@ import (
type ( type (
Kafka struct { Kafka struct {
logger *zap.Logger
routerUrl string routerUrl string
brokers []string brokers []string
version sarama.KafkaVersion version sarama.KafkaVersion
} }
) )
func makeKafkaMessageQueue(routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) { func makeKafkaMessageQueue(logger *zap.Logger, routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) {
if len(routerUrl) == 0 || len(mqCfg.Url) == 0 { if len(routerUrl) == 0 || len(mqCfg.Url) == 0 {
return nil, errors.New("The router URL or MQ URL is empty") return nil, errors.New("the router URL or MQ URL is empty")
} }
mqKafkaVersion := os.Getenv("MESSAGE_QUEUE_KAFKA_VERSION") mqKafkaVersion := os.Getenv("MESSAGE_QUEUE_KAFKA_VERSION")
// Parse version string // Parse version string
kafkaVersion, err := sarama.ParseKafkaVersion(mqKafkaVersion) kafkaVersion, err := sarama.ParseKafkaVersion(mqKafkaVersion)
if err != nil { if err != nil {
log.Warningf("Error parsing version string %q: %v. Falling back to %q", mqKafkaVersion, err, kafkaVersion) logger.Warn("error parsing kafka version string - falling back to default",
zap.Error(err),
zap.String("failed_version", mqKafkaVersion),
zap.Any("default_version", kafkaVersion))
} }
kafka := Kafka{ kafka := Kafka{
logger: logger.Named("kafka"),
routerUrl: routerUrl, routerUrl: routerUrl,
brokers: strings.Split(mqCfg.Url, ","), brokers: strings.Split(mqCfg.Url, ","),
version: kafkaVersion, version: kafkaVersion,
} }
log.Infof("Created Queue %v", kafka) logger.Info("created kafka queue", zap.Any("kafka", kafka))
return kafka, nil return kafka, nil
} }
@@ -66,8 +71,8 @@ func isTopicValidForKafka(topic string) bool {
} }
func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscription, error) { func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscription, error) {
log.Infof("Inside kakfa subscribe %q", trigger) kafka.logger.Info("inside kakfa subscribe", zap.Any("trigger", trigger))
log.Infof("brokers set to %q", kafka.brokers) kafka.logger.Info("brokers set", zap.Strings("brokers", kafka.brokers))
// Create new consumer // Create new consumer
consumerConfig := cluster.NewConfig() consumerConfig := cluster.NewConfig()
@@ -75,7 +80,7 @@ func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubs
consumerConfig.Group.Return.Notifications = true consumerConfig.Group.Return.Notifications = true
consumerConfig.Config.Version = kafka.version consumerConfig.Config.Version = kafka.version
consumer, err := cluster.NewConsumer(kafka.brokers, string(trigger.Metadata.UID), []string{trigger.Spec.Topic}, consumerConfig) consumer, err := cluster.NewConsumer(kafka.brokers, string(trigger.Metadata.UID), []string{trigger.Spec.Topic}, consumerConfig)
log.Infof("Created a new consumer: %#v", consumer) kafka.logger.Info("created a new consumer", zap.Any("consumer", consumer))
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -87,7 +92,7 @@ func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubs
producerConfig.Producer.Return.Successes = true producerConfig.Producer.Return.Successes = true
producerConfig.Version = kafka.version producerConfig.Version = kafka.version
producer, err := sarama.NewSyncProducer(kafka.brokers, producerConfig) producer, err := sarama.NewSyncProducer(kafka.brokers, producerConfig)
log.Infof("Created a new producer %q", producer) kafka.logger.Info("created a new producer", zap.Any("consumer", producer))
if err != nil { if err != nil {
panic(err) panic(err)
} }
@@ -95,21 +100,21 @@ func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubs
// consume errors // consume errors
go func() { go func() {
for err := range consumer.Errors() { for err := range consumer.Errors() {
log.Printf("Error: %s\n", err.Error()) kafka.logger.Error("consumer error", zap.Error(err))
} }
}() }()
// consume notifications // consume notifications
go func() { go func() {
for ntf := range consumer.Notifications() { for ntf := range consumer.Notifications() {
log.Printf("Rebalanced: %+v\n", ntf) kafka.logger.Info("consumer notification", zap.Any("notification", ntf))
} }
}() }()
// consume messages // consume messages
go func() { go func() {
for msg := range consumer.Messages() { for msg := range consumer.Messages() {
log.Infof("Calling message handler with value " + string(msg.Value[:])) kafka.logger.Info("calling message handler", zap.String("message", string(msg.Value[:])))
if kafkaMsgHandler(&kafka, producer, trigger, msg) { if kafkaMsgHandler(&kafka, producer, trigger, msg) {
consumer.MarkOffset(msg, "") // mark message as processed consumer.MarkOffset(msg, "") // mark message as processed
} }
@@ -127,12 +132,13 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
var value string = string(msg.Value[:]) var value string = string(msg.Value[:])
// Support other function ref types // Support other function ref types
if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName {
log.Fatalf("Unsupported function reference type (%v) for trigger %v", kafka.logger.Fatal("unsupported function reference type for trigger",
trigger.Spec.FunctionReference.Type, trigger.Metadata.Name) zap.Any("function_reference_type", trigger.Spec.FunctionReference.Type),
zap.String("trigger", trigger.Metadata.Name))
} }
url := kafka.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name, trigger.Metadata.Namespace), "/") url := kafka.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name, trigger.Metadata.Namespace), "/")
log.Printf("Making HTTP request to %v", url) kafka.logger.Info("making HTTP request", zap.String("url", url))
// Generate the Headers // Generate the Headers
fissionHeaders := map[string]string{ fissionHeaders := map[string]string{
@@ -145,7 +151,9 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
// Create request // Create request
req, err := http.NewRequest("POST", url, strings.NewReader(value)) req, err := http.NewRequest("POST", url, strings.NewReader(value))
if err != nil { if err != nil {
log.Warningf("Request creation failed: %v", url) kafka.logger.Error("failed to create HTTP request to invoke function",
zap.Error(err),
zap.String("function_url", url))
return false return false
} }
@@ -156,7 +164,8 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
req.Header.Add(string(h.Key), string(h.Value)) req.Header.Add(string(h.Key), string(h.Value))
} }
} else { } else {
log.Warningf("Headers are not supported by Kafka version %q, needs v0.11+: no record headers to add in HTTP request", kafka.version) kafka.logger.Warn("headers are not supported by current Kafka version, needs v0.11+: no record headers to add in HTTP request",
zap.Any("current_version", kafka.version))
} }
for k, v := range fissionHeaders { for k, v := range fissionHeaders {
@@ -169,7 +178,10 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
// Make the request // Make the request
resp, err = http.DefaultClient.Do(req) resp, err = http.DefaultClient.Do(req)
if err != nil { if err != nil {
log.Errorf("Error invoking function for trigger %v: %v", trigger.Metadata.Name, err) kafka.logger.Error("sending function invocation request failed",
zap.Error(err),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
continue continue
} }
if resp == nil { if resp == nil {
@@ -182,18 +194,23 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
} }
if resp == nil { if resp == nil {
log.Warning("Every retry failed; final retry gave empty response.") kafka.logger.Warn("every function invocation retry failed; final retry gave empty response",
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
return false return false
} }
defer resp.Body.Close() defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
log.Infof("Got response " + string(body)) kafka.logger.Info("got response from function invocation",
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name),
zap.String("body", string(body)))
if err != nil { if err != nil {
errorHandler(trigger, producer, fmt.Sprintf("Request body error: %v", string(body))) errorHandler(kafka.logger, trigger, producer, fmt.Sprintf("request body error: %v", string(body)))
return false return false
} }
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
errorHandler(trigger, producer, fmt.Sprintf("Request returned failure: %v", resp.StatusCode)) errorHandler(kafka.logger, trigger, producer, fmt.Sprintf("request returned failure: %v", resp.StatusCode))
return false return false
} }
if len(trigger.Spec.ResponseTopic) > 0 { if len(trigger.Spec.ResponseTopic) > 0 {
@@ -207,7 +224,8 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
} }
} }
} else { } else {
log.Warningf("Headers are not supported by Kafka version %q, needs v0.11+: dropping the headers", kafka.version) kafka.logger.Warn("headers are not supported by current Kafka version, needs v0.11+: no record headers to add in HTTP request",
zap.Any("current_version", kafka.version))
} }
_, _, err := producer.SendMessage(&sarama.ProducerMessage{ _, _, err := producer.SendMessage(&sarama.ProducerMessage{
@@ -216,24 +234,30 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
Headers: kafkaRecordHeaders, Headers: kafkaRecordHeaders,
}) })
if err != nil { if err != nil {
log.Warningf("Failed to publish message to topic %s: %v", trigger.Spec.ResponseTopic, err) kafka.logger.Warn("failed to publish response body from function invocation to topic",
zap.Error(err),
zap.String("topic", trigger.Spec.Topic),
zap.String("function_url", url))
return false return false
} }
} }
return true return true
} }
func errorHandler(trigger *crd.MessageQueueTrigger, producer sarama.SyncProducer, body string) { func errorHandler(logger *zap.Logger, trigger *crd.MessageQueueTrigger, producer sarama.SyncProducer, body string) {
if len(trigger.Spec.ErrorTopic) > 0 { if len(trigger.Spec.ErrorTopic) > 0 {
_, _, err := producer.SendMessage(&sarama.ProducerMessage{ _, _, err := producer.SendMessage(&sarama.ProducerMessage{
Topic: trigger.Spec.ErrorTopic, Topic: trigger.Spec.ErrorTopic,
Value: sarama.StringEncoder(body), Value: sarama.StringEncoder(body),
}) })
if err != nil { if err != nil {
log.Warningf("Failed to publish message to error topic %s: %v", trigger.Spec.ErrorTopic, err) logger.Warn("failed to publish message to error topic",
zap.Error(err),
zap.String("topic", trigger.Spec.Topic))
return return
} }
} else { } else {
log.Printf(body) logger.Error("message received to publish to error topic, but no error topic was set",
zap.String("message", body))
} }
} }
mqtrigger/messageQueue/messageQueue.go
+18 -15
View File
@@ -18,9 +18,10 @@ package messageQueue
import ( import (
"errors" "errors"
"fmt"
"time" "time"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
@@ -50,6 +51,7 @@ type (
} }
MessageQueueTriggerManager struct { MessageQueueTriggerManager struct {
logger *zap.Logger
reqChan chan request reqChan chan request
mqCfg MessageQueueConfig mqCfg MessageQueueConfig
triggers map[string]*triggerSubscription triggers map[string]*triggerSubscription
@@ -73,27 +75,28 @@ type (
} }
) )
func MakeMessageQueueTriggerManager(fissionClient *crd.FissionClient, routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager { func MakeMessageQueueTriggerManager(logger *zap.Logger, fissionClient *crd.FissionClient, routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager {
var messageQueue MessageQueue var messageQueue MessageQueue
var err error var err error
mqTriggerMgr := MessageQueueTriggerManager{ mqTriggerMgr := MessageQueueTriggerManager{
logger: logger.Named("message_queue_trigger_manager"),
reqChan: make(chan request), reqChan: make(chan request),
triggers: make(map[string]*triggerSubscription), triggers: make(map[string]*triggerSubscription),
fissionClient: fissionClient, fissionClient: fissionClient,
} }
switch mqConfig.MQType { switch mqConfig.MQType {
case fission.MessageQueueTypeNats: case fission.MessageQueueTypeNats:
messageQueue, err = makeNatsMessageQueue(routerUrl, mqConfig) messageQueue, err = makeNatsMessageQueue(logger, routerUrl, mqConfig)
case fission.MessageQueueTypeASQ: case fission.MessageQueueTypeASQ:
messageQueue, err = newAzureStorageConnection(routerUrl, mqConfig) messageQueue, err = newAzureStorageConnection(logger, routerUrl, mqConfig)
case fission.MessageQueueTypeKafka: case fission.MessageQueueTypeKafka:
messageQueue, err = makeKafkaMessageQueue(routerUrl, mqConfig) messageQueue, err = makeKafkaMessageQueue(logger, routerUrl, mqConfig)
default: default:
err = errors.New("No matched message queue type found") err = fmt.Errorf("no supported message queue type found for %q", mqConfig.MQType)
} }
if err != nil { if err != nil {
log.Fatalf("Failed to connect to remote message queue server: %v", err) logger.Fatal("failed to connect to remote message queue server", zap.Error(err))
} }
mqTriggerMgr.messageQueue = messageQueue mqTriggerMgr.messageQueue = messageQueue
go mqTriggerMgr.service() go mqTriggerMgr.service()
@@ -109,7 +112,7 @@ func (mqt *MessageQueueTriggerManager) service() {
var err error var err error
k := crd.CacheKey(&req.triggerSub.trigger.Metadata) k := crd.CacheKey(&req.triggerSub.trigger.Metadata)
if _, ok := mqt.triggers[k]; ok { if _, ok := mqt.triggers[k]; ok {
err = errors.New("Trigger already exists") err = errors.New("trigger already exists")
} else { } else {
mqt.triggers[k] = req.triggerSub mqt.triggers[k] = req.triggerSub
} }
@@ -164,11 +167,11 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
newTriggers, err := mqt.fissionClient.MessageQueueTriggers(metav1.NamespaceAll).List(metav1.ListOptions{}) newTriggers, err := mqt.fissionClient.MessageQueueTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil { if err != nil {
if fission.IsNetworkError(err) { if fission.IsNetworkError(err) {
log.Printf("Encounter network error, retry again: %v", err) mqt.logger.Info("encountered network error, will retry", zap.Error(err))
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
continue continue
} }
log.Fatalf("Failed to read message queue trigger list: %v", err) mqt.logger.Fatal("failed to read message queue trigger list", zap.Error(err))
} }
newTriggerMap := make(map[string]*crd.MessageQueueTrigger) newTriggerMap := make(map[string]*crd.MessageQueueTrigger)
for index := range newTriggers.Items { for index := range newTriggers.Items {
@@ -188,7 +191,7 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
// actually subscribe using the message queue client impl // actually subscribe using the message queue client impl
sub, err := mqt.messageQueue.subscribe(trigger) sub, err := mqt.messageQueue.subscribe(trigger)
if err != nil { if err != nil {
log.Warnf("Failed to subscribe to message queue trigger %s: %v", trigger.Metadata.Name, err) mqt.logger.Warn("failed to subscribe to message queue trigger", zap.Error(err), zap.String("trigger_name", trigger.Metadata.Name))
continue continue
} }
@@ -200,10 +203,10 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
// add to our list // add to our list
err = mqt.addTrigger(&triggerSub) err = mqt.addTrigger(&triggerSub)
if err != nil { if err != nil {
log.Fatalf("Message queue trigger %s addition failed: %v", trigger.Metadata.Name, err) mqt.logger.Fatal("adding message queue trigger failed", zap.Error(err), zap.String("trigger_name", trigger.Metadata.Name))
} }
log.Infof("Message queue trigger %s created", trigger.Metadata.Name) mqt.logger.Info("message queue trigger created", zap.String("trigger_name", trigger.Metadata.Name))
} }
// remove old triggers // remove old triggers
@@ -213,11 +216,11 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
} }
err := mqt.messageQueue.unsubscribe(triggerSub.subscription) err := mqt.messageQueue.unsubscribe(triggerSub.subscription)
if err != nil { if err != nil {
log.Warnf("Failed to unsubscribe to trigger %s: %v", triggerSub.trigger.Metadata.Name, err) mqt.logger.Warn("failed to unsubscribe from message queue trigger", zap.Error(err), zap.String("trigger_name", triggerSub.trigger.Metadata.Name))
continue continue
} }
mqt.delTrigger(&triggerSub.trigger.Metadata) mqt.delTrigger(&triggerSub.trigger.Metadata)
log.Infof("Message queue trigger %s deleted", triggerSub.trigger.Metadata.Name) mqt.logger.Info("message queue trigger deleted", zap.String("trigger_name", triggerSub.trigger.Metadata.Name))
} }
// TODO replace with a watch // TODO replace with a watch
mqtrigger/messageQueue/nats.go
+36 -14
View File
@@ -18,7 +18,6 @@ package messageQueue
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@@ -26,7 +25,7 @@ import (
ns "github.com/nats-io/go-nats-streaming" ns "github.com/nats-io/go-nats-streaming"
nsUtil "github.com/nats-io/nats-streaming-server/util" nsUtil "github.com/nats-io/nats-streaming-server/util"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
@@ -41,17 +40,19 @@ const (
type ( type (
Nats struct { Nats struct {
logger *zap.Logger
nsConn ns.Conn nsConn ns.Conn
routerUrl string routerUrl string
} }
) )
func makeNatsMessageQueue(routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) { func makeNatsMessageQueue(logger *zap.Logger, routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) {
conn, err := ns.Connect(natsClusterID, natsClientID, ns.NatsURL(mqCfg.Url)) conn, err := ns.Connect(natsClusterID, natsClientID, ns.NatsURL(mqCfg.Url))
if err != nil { if err != nil {
return nil, err return nil, err
} }
nats := Nats{ nats := Nats{
logger: logger.Named("nats"),
nsConn: conn, nsConn: conn,
routerUrl: routerUrl, routerUrl: routerUrl,
} }
@@ -62,7 +63,7 @@ func (nats Nats) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscr
subj := trigger.Spec.Topic subj := trigger.Spec.Topic
if !isTopicValidForNats(subj) { if !isTopicValidForNats(subj) {
return nil, errors.New(fmt.Sprintf("Not a valid topic: %s", trigger.Spec.Topic)) return nil, fmt.Errorf("not a valid topic: %q", trigger.Spec.Topic)
} }
opts := []ns.SubscriptionOption{ opts := []ns.SubscriptionOption{
@@ -96,15 +97,16 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
// Support other function ref types // Support other function ref types
if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName { if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName {
log.Fatalf("Unsupported function reference type (%v) for trigger %v", nats.logger.Fatal("unsupported function reference type for trigger",
trigger.Spec.FunctionReference.Type, trigger.Metadata.Name) zap.Any("function_reference_type", trigger.Spec.FunctionReference.Type),
zap.String("trigger", trigger.Metadata.Name))
} }
// with the addition of multi-tenancy, the users can create functions in any namespace. however, // with the addition of multi-tenancy, the users can create functions in any namespace. however,
// the triggers can only be created in the same namespace as the function. // the triggers can only be created in the same namespace as the function.
// so essentially, function namespace = trigger namespace. // so essentially, function namespace = trigger namespace.
url := nats.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name, trigger.Metadata.Namespace), "/") url := nats.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name, trigger.Metadata.Namespace), "/")
log.Printf("Making HTTP request to %v", url) nats.logger.Info("making HTTP request", zap.String("url", url))
headers := map[string]string{ headers := map[string]string{
"X-Fission-MQTrigger-Topic": trigger.Spec.Topic, "X-Fission-MQTrigger-Topic": trigger.Spec.Topic,
@@ -117,7 +119,9 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
req, err := http.NewRequest("POST", url, bytes.NewReader(msg.Data)) req, err := http.NewRequest("POST", url, bytes.NewReader(msg.Data))
if err != nil { if err != nil {
log.Errorf("Could not issue POST request with message to url %v", url) nats.logger.Error("failed to create HTTP request to invoke function",
zap.Error(err),
zap.String("function_url", url))
return return
} }
@@ -130,7 +134,10 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
// Make the request // Make the request
resp, err = http.DefaultClient.Do(req) resp, err = http.DefaultClient.Do(req)
if err != nil { if err != nil {
log.Errorf("Error invoking function for trigger %v: %v", trigger.Metadata.Name, err) nats.logger.Error("sending function invocation request failed",
zap.Error(err),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
continue continue
} }
if resp == nil { if resp == nil {
@@ -143,7 +150,9 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
} }
if resp == nil { if resp == nil {
log.Warning("Every retry failed; final retry gave empty response.") nats.logger.Warn("every function invocation retry failed; final retry gave empty response",
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
return return
} }
@@ -151,7 +160,10 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
body, bodyErr := ioutil.ReadAll(resp.Body) body, bodyErr := ioutil.ReadAll(resp.Body)
if bodyErr != nil { if bodyErr != nil {
log.Warningf("Response body error: %v", bodyErr) nats.logger.Error("error reading function invocation response",
zap.Error(err),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
return return
} }
@@ -160,7 +172,11 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
if len(trigger.Spec.ErrorTopic) > 0 && len(body) > 0 { if len(trigger.Spec.ErrorTopic) > 0 && len(body) > 0 {
publishErr := nats.nsConn.Publish(trigger.Spec.ErrorTopic, body) publishErr := nats.nsConn.Publish(trigger.Spec.ErrorTopic, body)
if publishErr != nil { if publishErr != nil {
log.Errorf("Failed to publish error to error topic: %v", publishErr) nats.logger.Error("failed to publish function invocation error to error topic",
zap.Error(publishErr),
zap.String("topic", trigger.Spec.ErrorTopic),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
// TODO: We will ack this message after max retries to prevent re-processing but // TODO: We will ack this message after max retries to prevent re-processing but
// this may cause message loss // this may cause message loss
} }
@@ -171,13 +187,19 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
// Trigger acks message only if a request was processed successfully // Trigger acks message only if a request was processed successfully
err = msg.Ack() err = msg.Ack()
if err != nil { if err != nil {
log.Warningf("Failed to ack message: %v", err) nats.logger.Error("failed to ack message after successful function invocation from trigger",
zap.Error(err),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
} }
if len(trigger.Spec.ResponseTopic) > 0 { if len(trigger.Spec.ResponseTopic) > 0 {
err = nats.nsConn.Publish(trigger.Spec.ResponseTopic, body) err = nats.nsConn.Publish(trigger.Spec.ResponseTopic, body)
if err != nil { if err != nil {
log.Warningf("Failed to publish message to topic %s: %v", trigger.Spec.ResponseTopic, err) nats.logger.Error("failed to publish message with function invocation response to topic",
zap.Error(err),
zap.String("topic", trigger.Spec.ResponseTopic),
zap.String("trigger", trigger.Metadata.Name))
} }
} }
} }
preupgradechecks/main.go
+13 -7
View File
@@ -17,10 +17,10 @@ limitations under the License.
package main package main
import ( import (
"os" "log"
"github.com/docopt/docopt-go" "github.com/docopt/docopt-go"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
) )
@@ -34,6 +34,12 @@ func getStringArgWithDefault(arg interface{}, defaultValue string) string {
} }
func main() { func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
usage := `Package to perform operations needed prior to fission installation usage := `Package to perform operations needed prior to fission installation
Usage: Usage:
pre-upgrade-checks --fn-pod-namespace=<podNamespace> --envbuilder-namespace=<envBuilderNamespace> pre-upgrade-checks --fn-pod-namespace=<podNamespace> --envbuilder-namespace=<envBuilderNamespace>
@@ -43,20 +49,20 @@ Options:
arguments, err := docopt.Parse(usage, nil, true, fission.BuildInfo().String(), false) arguments, err := docopt.Parse(usage, nil, true, fission.BuildInfo().String(), false)
if err != nil { if err != nil {
log.Fatalf("Error: %v", err) logger.Fatal("Could not parse command line arguments", zap.Error(err))
} }
functionPodNs := getStringArgWithDefault(arguments["--fn-pod-namespace"], "fission-function") functionPodNs := getStringArgWithDefault(arguments["--fn-pod-namespace"], "fission-function")
envBuilderNs := getStringArgWithDefault(arguments["--envbuilder-namespace"], "fission-builder") envBuilderNs := getStringArgWithDefault(arguments["--envbuilder-namespace"], "fission-builder")
crdBackedClient, err := makePreUpgradeTaskClient(functionPodNs, envBuilderNs) crdBackedClient, err := makePreUpgradeTaskClient(logger, functionPodNs, envBuilderNs)
if err != nil { if err != nil {
log.Printf("Error creating a crd client : %v, please retry helm upgrade", err) logger.Fatal("error creating a crd client, please retry helm upgrade",
os.Exit(1) zap.Error(err))
} }
if !crdBackedClient.IsFissionReInstall() { if !crdBackedClient.IsFissionReInstall() {
log.Printf("Nothing to do since CRDs are not present on the cluster") logger.Info("nothing to do since CRDs are not present on the cluster")
return return
} }
preupgradechecks/preupgradechecks.go
+39 -26
View File
@@ -18,10 +18,10 @@ package main
import ( import (
"fmt" "fmt"
"os"
multierror "github.com/hashicorp/go-multierror" multierror "github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus" "github.com/pkg/errors"
"go.uber.org/zap"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
k8serrors "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -33,6 +33,7 @@ import (
type ( type (
PreUpgradeTaskClient struct { PreUpgradeTaskClient struct {
logger *zap.Logger
fissionClient *crd.FissionClient fissionClient *crd.FissionClient
k8sClient *kubernetes.Clientset k8sClient *kubernetes.Clientset
apiExtClient *apiextensionsclient.Clientset apiExtClient *apiextensionsclient.Clientset
@@ -46,19 +47,14 @@ const (
FunctionCRD = "functions.fission.io" FunctionCRD = "functions.fission.io"
) )
func fatal(msg string) { func makePreUpgradeTaskClient(logger *zap.Logger, fnPodNs, envBuilderNs string) (*PreUpgradeTaskClient, error) {
os.Stderr.WriteString(msg + "\n")
os.Exit(1)
}
func makePreUpgradeTaskClient(fnPodNs, envBuilderNs string) (*PreUpgradeTaskClient, error) {
fissionClient, k8sClient, apiExtClient, err := crd.MakeFissionClient() fissionClient, k8sClient, apiExtClient, err := crd.MakeFissionClient()
if err != nil { if err != nil {
log.Errorf("Error making fission client") return nil, errors.Wrap(err, "error making fission client")
return nil, err
} }
return &PreUpgradeTaskClient{ return &PreUpgradeTaskClient{
logger: logger.Named("pre_upgrade_task_client"),
fissionClient: fissionClient, fissionClient: fissionClient,
k8sClient: k8sClient, k8sClient: k8sClient,
fnPodNs: fnPodNs, fnPodNs: fnPodNs,
@@ -86,7 +82,7 @@ func (client *PreUpgradeTaskClient) IsFissionReInstall() bool {
// VerifyFunctionSpecReferences verifies that a function references secrets, configmaps, pkgs in its own namespace and // VerifyFunctionSpecReferences verifies that a function references secrets, configmaps, pkgs in its own namespace and
// outputs a list of functions that don't adhere to this requirement. // outputs a list of functions that don't adhere to this requirement.
func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() { func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() {
log.Printf("Verifying Function spec references for all functions in the cluster") client.logger.Info("verifying function spec references for all functions in the cluster")
var result *multierror.Error var result *multierror.Error
var err error var err error
@@ -100,7 +96,9 @@ func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() {
} }
if err != nil { if err != nil {
fatal(fmt.Sprintf("Error: %v listing functions even after %d retries", err, MaxRetries)) client.logger.Fatal("error listing functions after max retries",
zap.Error(err),
zap.Int("max_retries", MaxRetries))
} }
// check that all secrets, configmaps, packages are in the same namespace // check that all secrets, configmaps, packages are in the same namespace
@@ -125,12 +123,12 @@ func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() {
} }
if result != nil { if result != nil {
log.Printf("Installation failed due to the following errors :") client.logger.Fatal("installation failed",
log.Printf("Summary : A function cannot reference secrets, configmaps and packages outside it's own namespace") zap.Error(err),
fatal(result.Error()) zap.String("summary", "a function cannot reference secrets, configmaps and packages outside it's own namespace"))
} }
log.Printf("Function Spec References verified") client.logger.Info("function spec references verified")
} }
// deleteClusterRoleBinding deletes the clusterRoleBinding passed as an argument to it. // deleteClusterRoleBinding deletes the clusterRoleBinding passed as an argument to it.
@@ -152,11 +150,13 @@ func (client *PreUpgradeTaskClient) RemoveClusterAdminRolesForFissionSAs() {
for _, clusterRoleBinding := range clusterRoleBindings { for _, clusterRoleBinding := range clusterRoleBindings {
err := client.deleteClusterRoleBinding(clusterRoleBinding) err := client.deleteClusterRoleBinding(clusterRoleBinding)
if err != nil { if err != nil {
fatal(fmt.Sprintf("Error deleting rolebinding : %s, err : %v", clusterRoleBinding, err)) client.logger.Fatal("error deleting rolebinding",
zap.Error(err),
zap.String("role_binding", clusterRoleBinding))
} }
} }
log.Println("Removed cluster admin privileges for fission-builder and fission-fetcher Service Accounts") client.logger.Info("femoved cluster admin privileges for fission-builder and fission-fetcher service accounts")
} }
// NeedRoleBindings checks if there is atleast one package or function in default namespace. // NeedRoleBindings checks if there is atleast one package or function in default namespace.
@@ -181,27 +181,40 @@ func (client *PreUpgradeTaskClient) NeedRoleBindings() bool {
// Setup appropriate role bindings for fission-fetcher and fission-builder SAs // Setup appropriate role bindings for fission-fetcher and fission-builder SAs
func (client *PreUpgradeTaskClient) SetupRoleBindings() { func (client *PreUpgradeTaskClient) SetupRoleBindings() {
if !client.NeedRoleBindings() { if !client.NeedRoleBindings() {
log.Printf("No fission objects found, so no role-bindings to create") client.logger.Info("no fission objects found, so no role-bindings to create")
return return
} }
// the fact that we're here implies that there had been a prior installation of fission and objects are present still // the fact that we're here implies that there had been a prior installation of fission and objects are present still
// so, we go ahead and create the role-bindings necessary for the fission-fetcher and fission-builder Service Accounts. // so, we go ahead and create the role-bindings necessary for the fission-fetcher and fission-builder Service Accounts.
err := fission.SetupRoleBinding(client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs) err := fission.SetupRoleBinding(client.logger, client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs)
if err != nil { if err != nil {
fatal(fmt.Sprintf("Error setting up rolebinding %s for %s.%s service account", fission.PackageGetterRB, fission.FissionFetcherSA, client.fnPodNs)) client.logger.Fatal("error setting up rolebinding for service account",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namespace", client.fnPodNs))
} }
err = fission.SetupRoleBinding(client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, client.envBuilderNs) err = fission.SetupRoleBinding(client.logger, client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, client.envBuilderNs)
if err != nil { if err != nil {
fatal(fmt.Sprintf("Error setting up rolebinding %s for %s.%s service account", fission.PackageGetterRB, fission.FissionBuilderSA, client.envBuilderNs)) client.logger.Fatal("error setting up rolebinding for service account",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("service_account", fission.FissionBuilderSA),
zap.String("service_account_namespace", client.envBuilderNs))
} }
err = fission.SetupRoleBinding(client.k8sClient, fission.SecretConfigMapGetterRB, metav1.NamespaceDefault, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs) err = fission.SetupRoleBinding(client.logger, client.k8sClient, fission.SecretConfigMapGetterRB, metav1.NamespaceDefault, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs)
if err != nil { if err != nil {
fatal(fmt.Sprintf("Error setting up rolebinding %s for %s.%s service account", fission.SecretConfigMapGetterRB, fission.FissionFetcherSA, client.fnPodNs)) client.logger.Fatal("error setting up rolebinding for service account",
zap.Error(err),
zap.String("role_binding", fission.SecretConfigMapGetterRB),
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namespace", client.fnPodNs))
} }
log.Printf("Created role-bindings : %s and %s in default namespace", fission.PackageGetterRB, fission.SecretConfigMapGetterRB) client.logger.Info("created rolebindings in default namespace",
zap.Strings("role_bindings", []string{fission.PackageGetterRB, fission.SecretConfigMapGetterRB}))
return return
} }
publisher/webhookPublisher.go
+27 -8
View File
@@ -19,15 +19,18 @@ package publisher
import ( import (
"bytes" "bytes"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"strings" "strings"
"time" "time"
"go.uber.org/zap"
) )
type ( type (
// A webhook publisher for a single URL. Satisifies the Publisher interface. // A webhook publisher for a single URL. Satisifies the Publisher interface.
WebhookPublisher struct { WebhookPublisher struct {
logger *zap.Logger
requestChannel chan *publishRequest requestChannel chan *publishRequest
maxRetries int maxRetries int
@@ -44,8 +47,9 @@ type (
} }
) )
func MakeWebhookPublisher(baseUrl string) *WebhookPublisher { func MakeWebhookPublisher(logger *zap.Logger, baseUrl string) *WebhookPublisher {
p := &WebhookPublisher{ p := &WebhookPublisher{
logger: logger.Named("webhook_publisher"),
baseUrl: baseUrl, baseUrl: baseUrl,
requestChannel: make(chan *publishRequest, 32), // buffered channel requestChannel: make(chan *publishRequest, 32), // buffered channel
// TODO make this configurable // TODO make this configurable
@@ -76,13 +80,16 @@ func (p *WebhookPublisher) svc() {
func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) { func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) {
url := p.baseUrl + "/" + strings.TrimPrefix(r.target, "/") url := p.baseUrl + "/" + strings.TrimPrefix(r.target, "/")
log.Printf("Making HTTP request to %v", url) p.logger.Info("making HTTP request", zap.String("url", url))
var buf bytes.Buffer var buf bytes.Buffer
buf.WriteString(r.body) buf.WriteString(r.body)
// Create request // Create request
req, err := http.NewRequest("POST", url, &buf) req, err := http.NewRequest("POST", url, &buf)
if err != nil {
p.logger.Error("error creating request", zap.Error(err), zap.String("url", url))
}
for k, v := range r.headers { for k, v := range r.headers {
req.Header.Set(k, v) req.Header.Set(k, v)
} }
@@ -98,13 +105,25 @@ func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) {
// Log errors // Log errors
if err != nil { if err != nil {
log.Printf("Request failed: %v", r) p.logger.Error("request failed",
zap.Error(err),
zap.Any("request", r),
zap.String("url", url))
} else if resp.StatusCode != 200 { } else if resp.StatusCode != 200 {
log.Printf("Request returned failure: %v", resp.StatusCode) p.logger.Error("request returned failure status code",
zap.Any("request", r),
zap.String("url", url),
zap.Int("status_code", resp.StatusCode))
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close() resp.Body.Close()
if err == nil { if err != nil {
log.Printf("request error: %v", string(body)) p.logger.Error("error reading error request body",
zap.Error(err),
zap.Any("request", r),
zap.String("url", url),
zap.Int("status_code", resp.StatusCode))
} else {
p.logger.Error("request error", zap.String("body", string(body)))
} }
} }
@@ -116,7 +135,7 @@ func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) {
p.requestChannel <- r p.requestChannel <- r
}) })
} else { } else {
log.Printf("Final retry failed, giving up on %v", url) p.logger.Error("final retry failed, giving up", zap.String("url", url))
// Event dropped // Event dropped
} }
} }
redis/redis.go
+18 -17
View File
@@ -26,30 +26,30 @@ import (
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/gomodule/redigo/redis" "github.com/gomodule/redigo/redis"
log "github.com/sirupsen/logrus" "github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/redis/build/gen" "github.com/fission/fission/redis/build/gen"
) )
func NewClient() redis.Conn { func NewClient() (redis.Conn, error) {
redisIP := os.Getenv("REDIS_SERVICE_HOST") // TODO: Do this here or somewhere earlier? redisIP := os.Getenv("REDIS_SERVICE_HOST") // TODO: Do this here or somewhere earlier?
redisPort := os.Getenv("REDIS_SERVICE_PORT") redisPort := os.Getenv("REDIS_SERVICE_PORT")
redisUrl := fmt.Sprintf("%s:%s", redisIP, redisPort)
if len(redisUrl) == 0 { if len(redisIP) == 0 || len(redisPort) == 0 {
log.Error("Could not reach Redis in cluster at IP ", redisUrl) return nil, errors.New("redis host or port not supplied")
return nil
} }
c, err := redis.Dial("tcp", redisUrl) redisURL := fmt.Sprintf("%s:%s", redisIP, redisPort)
c, err := redis.Dial("tcp", redisURL)
if err != nil { if err != nil {
log.Errorf("Could not connect to Redis: %v\n", err) return nil, errors.Wrapf(err, "could not connect to Redis at url %q", redisURL)
return nil
} }
return c return c, nil
} }
func Record(triggerName string, recorderName string, reqUID string, request *http.Request, originalUrl url.URL, payload string, response *http.Response, namespace string, timestamp int64) { func Record(logger *zap.Logger, triggerName string, recorderName string, reqUID string, request *http.Request, originalUrl url.URL, payload string, response *http.Response, namespace string, timestamp int64) {
// Case where the function should not have been recorded // Case where the function should not have been recorded
if len(reqUID) == 0 { if len(reqUID) == 0 {
return return
@@ -58,8 +58,9 @@ func Record(triggerName string, recorderName string, reqUID string, request *htt
fullPath := originalUrl.String() fullPath := originalUrl.String()
escPayload := string(json.RawMessage(payload)) escPayload := string(json.RawMessage(payload))
client := NewClient() client, err := NewClient()
if client == nil { if err != nil {
logger.Error("could not create redis client", zap.Error(err))
return return
} }
@@ -92,7 +93,7 @@ func Record(triggerName string, recorderName string, reqUID string, request *htt
PostForm: postForm, PostForm: postForm,
} }
log.Info("Storing request > ", req) logger.Info("storing request", zap.Any("request", req))
resp := &redisCache.Response{ resp := &redisCache.Response{
Status: response.Status, Status: response.Status,
@@ -107,19 +108,19 @@ func Record(triggerName string, recorderName string, reqUID string, request *htt
data, err := proto.Marshal(ureq) data, err := proto.Marshal(ureq)
if err != nil { if err != nil {
log.Error("Error marshalling request: ", err) logger.Error("error marshalling request", zap.Error(err))
return return
} }
_, err = client.Do("HMSET", reqUID, "ReqResponse", data, "Timestamp", timestamp, "Trigger", triggerName) _, err = client.Do("HMSET", reqUID, "ReqResponse", data, "Timestamp", timestamp, "Trigger", triggerName)
if err != nil { if err != nil {
log.Error("Error saving request: ", err) logger.Error("error saving request", zap.Error(err))
return return
} }
_, err = client.Do("LPUSH", recorderName, reqUID) _, err = client.Do("LPUSH", recorderName, reqUID)
if err != nil { if err != nil {
log.Error("Error saving recorder-request pair: ", err) logger.Error("error saving recorder-request pair", zap.Error(err))
return return
} }
} }
redis/redisApi.go
+47 -44
View File
@@ -19,7 +19,6 @@ package redis
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
@@ -27,18 +26,20 @@ import (
"strings" "strings"
"time" "time"
"go.uber.org/zap"
"github.com/golang/protobuf/proto" "github.com/golang/protobuf/proto"
"github.com/gomodule/redigo/redis" "github.com/gomodule/redigo/redis"
log "github.com/sirupsen/logrus" "github.com/pkg/errors"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
"github.com/fission/fission/redis/build/gen" "github.com/fission/fission/redis/build/gen"
) )
func RecordsListAll() ([]byte, error) { func RecordsListAll(logger *zap.Logger) ([]byte, error) {
client := NewClient() client, err := NewClient()
if client == nil { if err != nil {
return nil, errors.New("failed to create redis client") return nil, errors.Wrap(err, "failed to create redis client")
} }
iter := 0 iter := 0
@@ -59,12 +60,12 @@ func RecordsListAll() ([]byte, error) {
if strings.HasPrefix(key, "REQ") { if strings.HasPrefix(key, "REQ") {
val, err := redis.Bytes(client.Do("HGET", key, "ReqResponse")) val, err := redis.Bytes(client.Do("HGET", key, "ReqResponse"))
if err != nil { if err != nil {
log.Error("Error retrieving request from Redis: ", err) logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err return nil, err
} }
entry, err := deserializeReqResponse(val, key) entry, err := deserializeReqResponse(val, key)
if err != nil { if err != nil {
log.Error("Error deserializing request: ", err) logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err return nil, err
} }
filtered = append(filtered, entry) filtered = append(filtered, entry)
@@ -84,18 +85,22 @@ func RecordsListAll() ([]byte, error) {
// Input: `from` (hours ago, between 0 [today] and 5) and `to` (same units) // Input: `from` (hours ago, between 0 [today] and 5) and `to` (same units)
// Note: Fractional values don't seem to work -- document that for the user // Note: Fractional values don't seem to work -- document that for the user
func RecordsFilterByTime(from string, to string) ([]byte, error) { func RecordsFilterByTime(logger *zap.Logger, from string, to string) ([]byte, error) {
rangeStart, rangeEnd, err := obtainInterval(from, to) rangeStart, rangeEnd, err := obtainInterval(from, to)
log.Debug("Interval inferred: ", rangeStart, rangeEnd) if err != nil {
if rangeStart >= rangeEnd {
log.Error("Invalid chronology")
return nil, err return nil, err
} }
logger.Debug("interval inferred", zap.Int64("range_start", rangeStart), zap.Int64("range_end", rangeEnd))
client := NewClient() if rangeStart >= rangeEnd {
e := "invalid chronology - start is greater than or equal to end"
logger.Error(e, zap.Int64("range_start", rangeStart), zap.Int64("range_end", rangeEnd))
return nil, errors.New(e)
}
client, err := NewClient()
if client == nil { if client == nil {
return nil, errors.New("failed to create redis client") return nil, errors.Wrap(err, "failed to create redis client")
} }
iter := 0 iter := 0
@@ -114,24 +119,24 @@ func RecordsFilterByTime(from string, to string) ([]byte, error) {
if strings.HasPrefix(key, "REQ") { if strings.HasPrefix(key, "REQ") {
val, err := redis.Strings(client.Do("HMGET", key, "Timestamp")) val, err := redis.Strings(client.Do("HMGET", key, "Timestamp"))
if err != nil { if err != nil {
log.Error("Error retrieving timestamp from Redis: ", err) logger.Error("error retrieving timestamp from redis", zap.Error(err))
return nil, err return nil, err
} }
tsO, err := strconv.Atoi(val[0]) tsO, err := strconv.Atoi(val[0])
if err != nil { if err != nil {
log.Error("Error converting timestamp to int: ", err) logger.Error("error converting timestamp to int", zap.Error(err))
return nil, err return nil, err
} }
ts := int64(tsO) ts := int64(tsO)
if ts >= rangeStart && ts <= rangeEnd { if ts >= rangeStart && ts <= rangeEnd {
val2, err := redis.Bytes(client.Do("HGET", key, "ReqResponse")) val2, err := redis.Bytes(client.Do("HGET", key, "ReqResponse"))
if err != nil { if err != nil {
log.Error("Error retrieving request from Redis: ", err) logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err return nil, err
} }
entry, err := deserializeReqResponse(val2, key) entry, err := deserializeReqResponse(val2, key)
if err != nil { if err != nil {
log.Error("Error deserializing request: ", err) logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err return nil, err
} }
filtered = append(filtered, entry) filtered = append(filtered, entry)
@@ -151,7 +156,7 @@ func RecordsFilterByTime(from string, to string) ([]byte, error) {
return resp, nil return resp, nil
} }
func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) { func RecordsFilterByTrigger(logger *zap.Logger, queriedTriggerName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) {
matchingRecorders := make(map[string]bool) matchingRecorders := make(map[string]bool)
// Implicit triggers: // Implicit triggers:
@@ -178,9 +183,9 @@ func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderLi
} }
} }
client := NewClient() client, err := NewClient()
if client == nil { if err != nil {
return nil, errors.New("failed to create redis client") return nil, errors.Wrap(err, "failed to create redis client")
} }
var filtered []*redisCache.RecordedEntry var filtered []*redisCache.RecordedEntry
@@ -195,19 +200,19 @@ func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderLi
for _, reqUID := range val { for _, reqUID := range val {
val, err := redis.Strings(client.Do("HMGET", reqUID, "Trigger")) // 1-to-1 reqUID - trigger? val, err := redis.Strings(client.Do("HMGET", reqUID, "Trigger")) // 1-to-1 reqUID - trigger?
if err != nil { if err != nil {
log.Error("Error retrieving trigger for a request from Redis: ", err) logger.Error("error retrieving trigger for a request from redis", zap.Error(err))
return nil, err return nil, err
} }
if val[0] == queriedTriggerName { if val[0] == queriedTriggerName {
// TODO: Reconsider multiple commands // TODO: Reconsider multiple commands
val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse")) val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse"))
if err != nil { if err != nil {
log.Error("Error retrieving request from Redis: ", err) logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err return nil, err
} }
entry, err := deserializeReqResponse(val, reqUID) entry, err := deserializeReqResponse(val, reqUID)
if err != nil { if err != nil {
log.Error("Error deserializing request: ", err) logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err return nil, err
} }
filtered = append(filtered, entry) filtered = append(filtered, entry)
@@ -222,7 +227,7 @@ func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderLi
return resp, nil return resp, nil
} }
func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) { func RecordsFilterByFunction(logger *zap.Logger, queriedFunctionName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) {
// Implicit functions: // Implicit functions:
// Sometimes functions are not explicitly attached to recorders but we still want to be able to // Sometimes functions are not explicitly attached to recorders but we still want to be able to
@@ -251,9 +256,9 @@ func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.Recorder
} }
} }
client := NewClient() client, err := NewClient()
if client == nil { if err != nil {
return nil, errors.New("failed to create redis client") return nil, errors.Wrap(err, "failed to create redis client")
} }
var filtered []*redisCache.RecordedEntry var filtered []*redisCache.RecordedEntry
@@ -273,12 +278,12 @@ func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.Recorder
if exists > 0 { if exists > 0 {
val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse")) val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse"))
if err != nil { if err != nil {
log.Error("Error retrieving request from Redis: ", err) logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err return nil, err
} }
entry, err := deserializeReqResponse(val, reqUID) entry, err := deserializeReqResponse(val, reqUID)
if err != nil { if err != nil {
log.Error("Error deserializing request: ", err) logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err return nil, err
} }
filtered = append(filtered, entry) filtered = append(filtered, entry)
@@ -298,10 +303,8 @@ func deserializeReqResponse(value []byte, reqUID string) (*redisCache.RecordedEn
data := &redisCache.UniqueRequest{} data := &redisCache.UniqueRequest{}
err := proto.Unmarshal(value, data) err := proto.Unmarshal(value, data)
if err != nil { if err != nil {
log.Error("Error unmarshalling request: ", err) return nil, errors.Wrap(err, "error unmarshalling request")
return nil, err
} }
log.Info("Parsed protobuf bytes: ", data)
transformed := &redisCache.RecordedEntry{ transformed := &redisCache.RecordedEntry{
ReqUID: reqUID, ReqUID: reqUID,
Req: data.Req, Req: data.Req,
@@ -341,38 +344,38 @@ func includesTrigger(triggers []string, query string) bool {
return false return false
} }
func ReplayByReqUID(routerUrl string, queriedID string) ([]byte, error) { func ReplayByReqUID(logger *zap.Logger, routerUrl string, queriedID string) ([]byte, error) {
client := NewClient() client, err := NewClient()
if client == nil { if err != nil {
return nil, errors.New("failed to create redis client") return nil, errors.Wrap(err, "failed to create redis client")
} }
exists, err := redis.Int(client.Do("EXISTS", queriedID)) exists, err := redis.Int(client.Do("EXISTS", queriedID))
if exists != 1 || err != nil { if exists != 1 || err != nil {
log.Error("couldn't find request to replay") logger.Error("could not find request to replay in redis", zap.Error(err))
return nil, err return nil, err
} }
val, err := redis.Bytes(client.Do("HGET", queriedID, "ReqResponse")) val, err := redis.Bytes(client.Do("HGET", queriedID, "ReqResponse"))
if err != nil { if err != nil {
log.Error("couldn't obtain ReqResponse for this ID") logger.Error("could not obtain ReqResponse for ID from redis", zap.Error(err), zap.String("id", queriedID))
return nil, err return nil, err
} }
entry, err := deserializeReqResponse(val, queriedID) entry, err := deserializeReqResponse(val, queriedID)
if err != nil { if err != nil {
log.Error("couldn't deserialize ReqResponse") logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err return nil, err
} }
replayed, err := ReplayRequest(routerUrl, entry.Req) replayed, err := ReplayRequest(routerUrl, entry.Req)
if err != nil { if err != nil {
log.Error("couldn't replay request") logger.Error("error replaying request", zap.Error(err))
return nil, err return nil, err
} }
resp, err := json.Marshal(replayed) resp, err := json.Marshal(replayed)
if err != nil { if err != nil {
log.Error("couldn't marshall replayed request response") logger.Error("error marshalling replayed request response", zap.Error(err))
return nil, err return nil, err
} }
router/functionHandler.go
+48 -24
View File
@@ -33,8 +33,8 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/satori/go.uuid" "github.com/satori/go.uuid"
log "github.com/sirupsen/logrus"
"go.opencensus.io/plugin/ochttp" "go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
@@ -51,6 +51,7 @@ const (
type ( type (
functionHandler struct { functionHandler struct {
logger *zap.Logger
fmap *functionServiceMap fmap *functionServiceMap
frmap *functionRecorderMap frmap *functionRecorderMap
trmap *triggerRecorderMap trmap *triggerRecorderMap
@@ -86,6 +87,7 @@ type (
// A layer on top of http.DefaultTransport, with retries. // A layer on top of http.DefaultTransport, with retries.
RetryingRoundTripper struct { RetryingRoundTripper struct {
logger *zap.Logger
funcHandler *functionHandler funcHandler *functionHandler
base http.RoundTripper base http.RoundTripper
} }
@@ -146,7 +148,7 @@ func (w *fakeCloseReadCloser) RealClose() error {
// if it returned an error. // if it returned an error.
func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) { func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
// Set forwarded host header if not exists // Set forwarded host header if not exists
addForwardedHostHeader(req) roundTripper.addForwardedHostHeader(req)
// TODO: Keep? --> Needed for queries encoded in URL before they're stripped by the proxy // TODO: Keep? --> Needed for queries encoded in URL before they're stripped by the proxy
var originalUrl url.URL var originalUrl url.URL
@@ -164,7 +166,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
rdr1.Read(p) rdr1.Read(p)
postedBody = string(p) postedBody = string(p)
log.Info(fmt.Sprintf("%v", postedBody)) roundTripper.logger.Info("roundtripper posted body", zap.String("body", postedBody))
req.Body = rdr2 req.Body = rdr2
} }
} }
@@ -287,13 +289,15 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
if roundTripper.funcHandler.httpTrigger != nil { if roundTripper.funcHandler.httpTrigger != nil {
trigger := roundTripper.funcHandler.httpTrigger.Metadata.Name trigger := roundTripper.funcHandler.httpTrigger.Metadata.Name
redis.Record( redis.Record(
roundTripper.logger,
trigger, trigger,
roundTripper.funcHandler.recorderName, roundTripper.funcHandler.recorderName,
req.Header.Get("X-Fission-ReqUID"), req, originalUrl, postedBody, resp, fnMeta.Namespace, req.Header.Get("X-Fission-ReqUID"), req, originalUrl, postedBody, resp, fnMeta.Namespace,
time.Now().UnixNano(), time.Now().UnixNano(),
) )
} else { } else {
log.Printf("No http trigger attached for recorder: %v", roundTripper.funcHandler.recorderName) roundTripper.logger.Error("no http trigger attached for recorder",
zap.String("recorder", roundTripper.funcHandler.recorderName))
} }
} }
@@ -303,7 +307,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
// if transport.RoundTrip returns a non-network dial error, then relay it back to user // if transport.RoundTrip returns a non-network dial error, then relay it back to user
if !fission.IsNetworkDialError(err) { if !fission.IsNetworkDialError(err) {
err = errors.Wrapf(err, "Error sending request to function %v", fnMeta.Name) err = errors.Wrapf(err, "error sending request to function %v", fnMeta.Name)
return resp, err return resp, err
} }
@@ -313,8 +317,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
executingTimeout = executingTimeout * time.Duration(roundTripper.funcHandler.tsRoundTripperParams.timeoutExponent) executingTimeout = executingTimeout * time.Duration(roundTripper.funcHandler.tsRoundTripperParams.timeoutExponent)
retryCounter++ retryCounter++
log.Printf("request to %s errored out. backing off for %v before retrying", roundTripper.logger.Info("request errored out - backing off before retrying",
req.URL.Host, executingTimeout) zap.String("url", req.URL.Host),
zap.Duration("backoff_timeout", executingTimeout))
time.Sleep(executingTimeout) time.Sleep(executingTimeout)
@@ -324,9 +329,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
} else { } else {
// if transport.RoundTrip returns a network dial error and serviceUrl was from cache, // if transport.RoundTrip returns a network dial error and serviceUrl was from cache,
// it means, the entry in router cache is stale, so invalidate it. // it means, the entry in router cache is stale, so invalidate it.
log.Printf("request to %s errored out. removing function : %s from router's cache "+ roundTripper.logger.Error("request errored out - removing function from router's cache and requesting a new service for function",
"and requesting a new service for function", zap.String("url", req.URL.Host),
req.URL.Host, fnMeta.Name) zap.String("function_name", fnMeta.Name))
roundTripper.funcHandler.fmap.remove(fnMeta) roundTripper.funcHandler.fmap.remove(fnMeta)
retryCounter = 0 retryCounter = 0
} }
@@ -340,7 +345,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
// finally, one more retry with the default timeout // finally, one more retry with the default timeout
resp, err = http.DefaultTransport.RoundTrip(req) resp, err = http.DefaultTransport.RoundTrip(req)
if err != nil { if err != nil {
log.Printf("Error getting response from function %v: %v", fnMeta.Name, err) roundTripper.logger.Error("error getting response from function",
zap.Error(err),
zap.String("function_name", fnMeta.Name))
} }
return resp, err return resp, err
@@ -385,19 +392,19 @@ func (fh functionHandler) handler(responseWriter http.ResponseWriter, request *h
UID := strings.ToLower(uuid.NewV4().String()) UID := strings.ToLower(uuid.NewV4().String())
reqUID = "REQ" + UID reqUID = "REQ" + UID
request.Header.Set("X-Fission-ReqUID", reqUID) request.Header.Set("X-Fission-ReqUID", reqUID)
log.Print("Record request with ReqUID: ", reqUID) fh.logger.Info("record request", zap.String("request_id", reqUID))
} }
if fh.httpTrigger != nil && fh.httpTrigger.Spec.FunctionReference.Type == fission.FunctionReferenceTypeFunctionWeights { if fh.httpTrigger != nil && fh.httpTrigger.Spec.FunctionReference.Type == fission.FunctionReferenceTypeFunctionWeights {
// canary deployment. need to determine the function to send request to now // canary deployment. need to determine the function to send request to now
fnMetadata := getCanaryBackend(fh.functionMetadataMap, fh.fnWeightDistributionList) fnMetadata := getCanaryBackend(fh.functionMetadataMap, fh.fnWeightDistributionList)
if fnMetadata == nil { if fnMetadata == nil {
log.Printf("Error getting canary backend ") fh.logger.Error("could not get canary backend", zap.String("request_id", reqUID))
// TODO : write error to responseWrite and return response // TODO : write error to responseWrite and return response
return return
} }
fh.function = fnMetadata fh.function = fnMetadata
log.Debugf("chosen fnBackend's metadata : %+v", fh.function) fh.logger.Debug("chosen function backend's metadata", zap.Any("metadata", fh.function))
} }
// system params // system params
@@ -413,6 +420,7 @@ func (fh functionHandler) handler(responseWriter http.ResponseWriter, request *h
proxy := &httputil.ReverseProxy{ proxy := &httputil.ReverseProxy{
Director: director, Director: director,
Transport: &RetryingRoundTripper{ Transport: &RetryingRoundTripper{
logger: fh.logger.Named("roundtripper"),
funcHandler: &fh, funcHandler: &fh,
base: &ochttp.Transport{ base: &ochttp.Transport{
Base: &http.Transport{ Base: &http.Transport{
@@ -471,7 +479,7 @@ func getCanaryBackend(fnMetadatamap map[string]*metav1.ObjectMeta, fnWtDistribut
} }
// addForwardedHostHeader add "forwarded host" to request header // addForwardedHostHeader add "forwarded host" to request header
func addForwardedHostHeader(req *http.Request) { func (roundTripper RetryingRoundTripper) addForwardedHostHeader(req *http.Request) {
// for more detailed information, please visit: // for more detailed information, please visit:
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
@@ -486,7 +494,9 @@ func addForwardedHostHeader(req *http.Request) {
reqUrl := fmt.Sprintf("%s://%s", req.Proto, req.Host) reqUrl := fmt.Sprintf("%s://%s", req.Proto, req.Host)
u, err := url.Parse(reqUrl) u, err := url.Parse(reqUrl)
if err != nil { if err != nil {
log.Printf("Error parsing request url (%v): %v", reqUrl, err) roundTripper.logger.Error("error parsing request url while adding forwarded host headers",
zap.Error(err),
zap.String("url", reqUrl))
return return
} }
@@ -531,14 +541,19 @@ func (fh *functionHandler) getServiceEntry(ctx context.Context) (serviceUrl *url
var u *url.URL var u *url.URL
// Get service entry from executor and update cache if its the first goroutine // Get service entry from executor and update cache if its the first goroutine
if firstToTheLock { // first to the service url if firstToTheLock { // first to the service url
log.Printf("Calling getServiceForFunction for function: %s", fh.function.Name) fh.logger.Info("calling getServiceForFunction",
zap.String("function_name", fh.function.Name))
u, err = fh.getServiceEntryFromExecutor(ctx) u, err = fh.getServiceEntryFromExecutor(ctx)
if err != nil { if err != nil {
log.Printf("Error getting service url from executor: %v", err) fh.logger.Error("error getting service url from executor",
zap.Error(err),
zap.String("function_name", fh.function.Name))
return nil, err return nil, err
} }
// add the address in router's cache // add the address in router's cache
log.Printf("Assigning service url: %s for function: %s", u, fh.function.Name) fh.logger.Info("assigning service url for function",
zap.String("url", u.String()),
zap.String("function_name", fh.function.Name))
fh.fmap.assign(fh.function, u) fh.fmap.assign(fh.function, u)
} else { } else {
u, err = fh.getServiceEntryFromCache() u, err = fh.getServiceEntryFromCache()
@@ -554,9 +569,12 @@ func (fh *functionHandler) getServiceEntry(ctx context.Context) (serviceUrl *url
}, },
) )
if err != nil { if err != nil {
err = errors.Wrap(err, fmt.Sprintf("Error updating service address entry for function %v_%v", fh.function.Name, fh.function.Namespace)) e := "error updating service address entry for function"
log.Println(err) fh.logger.Error(e,
return nil, false, err zap.Error(err),
zap.String("function_name", fh.function.Name),
zap.String("function_namespace", fh.function.Namespace))
return nil, false, errors.Wrapf(err, "%s %s_%s", e, fh.function.Name, fh.function.Namespace)
} }
record, ok := recordObj.(svcEntryRecord) record, ok := recordObj.(svcEntryRecord)
@@ -596,14 +614,20 @@ func (fh *functionHandler) getServiceEntryFromExecutor(ctx context.Context) (*ur
service, err := fh.executor.GetServiceForFunction(ctx, fh.function) service, err := fh.executor.GetServiceForFunction(ctx, fh.function)
if err != nil { if err != nil {
statusCode, errMsg := fission.GetHTTPError(err) statusCode, errMsg := fission.GetHTTPError(err)
log.Printf("Error from GetServiceForFunction for function (%v): %v : %v", fh.function, statusCode, errMsg) fh.logger.Error("error from GetServiceForFunction",
zap.Error(err),
zap.String("error_message", errMsg),
zap.Any("function", fh.function),
zap.Int("status_code", statusCode))
return nil, err return nil, err
} }
// parse the address into url // parse the address into url
serviceUrl, err := url.Parse(fmt.Sprintf("http://%v", service)) serviceUrl, err := url.Parse(fmt.Sprintf("http://%v", service))
if err != nil { if err != nil {
log.Printf("Error parsing service url (%v): %v", serviceUrl, err) fh.logger.Error("error parsing service url",
zap.Error(err),
zap.String("service_url", serviceUrl.String()))
return nil, err return nil, err
} }
router/functionHandler_test.go
+9 -3
View File
@@ -24,9 +24,11 @@ import (
"testing" "testing"
"time" "time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
func createBackendService(testResponseString string) *url.URL { func createBackendService(testResponseString string) *url.URL {
@@ -53,8 +55,10 @@ func TestFunctionProxying(t *testing.T) {
log.Printf("Created backend svc at %v", backendURL) log.Printf("Created backend svc at %v", backendURL)
fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault} fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault}
logger, err := zap.NewDevelopment()
panicIf(err)
fmap := makeFunctionServiceMap(0) fmap := makeFunctionServiceMap(logger, 0)
fmap.assign(fn, backendURL) fmap.assign(fn, backendURL)
httpTrigger := &crd.HTTPTrigger{ httpTrigger := &crd.HTTPTrigger{
@@ -70,7 +74,9 @@ func TestFunctionProxying(t *testing.T) {
}, },
} }
fh := &functionHandler{fmap: fmap, fh := &functionHandler{
logger: logger,
fmap: fmap,
function: fn, function: fn,
tsRoundTripperParams: &tsRoundTripperParams{ tsRoundTripperParams: &tsRoundTripperParams{
timeout: 50 * time.Millisecond, timeout: 50 * time.Millisecond,
router/functionRecorderMap.go
+8 -5
View File
@@ -17,9 +17,10 @@ limitations under the License.
package router package router
import ( import (
"log"
"time" "time"
"go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/cache" "github.com/fission/fission/cache"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
@@ -27,14 +28,16 @@ import (
type ( type (
functionRecorderMap struct { functionRecorderMap struct {
cache *cache.Cache // map[string]*crd.Recorder logger *zap.Logger
cache *cache.Cache // map[string]*crd.Recorder
} }
) )
// Why do we need an expiry? // Why do we need an expiry?
func makeFunctionRecorderMap(expiry time.Duration) *functionRecorderMap { func makeFunctionRecorderMap(logger *zap.Logger, expiry time.Duration) *functionRecorderMap {
return &functionRecorderMap{ return &functionRecorderMap{
cache: cache.MakeCache(expiry, 0), logger: logger.Named("function_recorder_map"),
cache: cache.MakeCache(expiry, 0),
} }
} }
@@ -53,7 +56,7 @@ func (frmap *functionRecorderMap) assign(function string, recorder *crd.Recorder
if e, ok := err.(fission.Error); ok && e.Code == fission.ErrorNameExists { if e, ok := err.(fission.Error); ok && e.Code == fission.ErrorNameExists {
return return
} }
log.Printf("error caching recorder for function name with a different value: %v", err) frmap.logger.Error("error caching recorder for function name with a different value", zap.Error(err))
} }
} }
router/functionServiceMap.go
+7 -5
View File
@@ -17,10 +17,10 @@ limitations under the License.
package router package router
import ( import (
"log"
"net/url" "net/url"
"time" "time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/cache" "github.com/fission/fission/cache"
@@ -28,7 +28,8 @@ import (
type ( type (
functionServiceMap struct { functionServiceMap struct {
cache *cache.Cache // map[metadataKey]*url.URL logger *zap.Logger
cache *cache.Cache // map[metadataKey]*url.URL
} }
// metav1.ObjectMeta is not hashable, so we make a hashable copy // metav1.ObjectMeta is not hashable, so we make a hashable copy
@@ -40,9 +41,10 @@ type (
} }
) )
func makeFunctionServiceMap(expiry time.Duration) *functionServiceMap { func makeFunctionServiceMap(logger *zap.Logger, expiry time.Duration) *functionServiceMap {
return &functionServiceMap{ return &functionServiceMap{
cache: cache.MakeCache(expiry, 0), logger: logger.Named("function_service_map"),
cache: cache.MakeCache(expiry, 0),
} }
} }
@@ -71,7 +73,7 @@ func (fmap *functionServiceMap) assign(f *metav1.ObjectMeta, serviceUrl *url.URL
if *serviceUrl == *(old.(*url.URL)) { if *serviceUrl == *(old.(*url.URL)) {
return return
} }
log.Printf("error caching service url for function with a different value: %v", err) fmap.logger.Error("error caching service url for function with a different value", zap.Error(err))
// ignore error // ignore error
} }
} }
router/functionServiceMap_test.go
+6 -1
View File
@@ -20,11 +20,16 @@ import (
"net/url" "net/url"
"testing" "testing"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
) )
func TestFunctionServiceMap(t *testing.T) { func TestFunctionServiceMap(t *testing.T) {
m := makeFunctionServiceMap(0) logger, err := zap.NewDevelopment()
panicIf(err)
m := makeFunctionServiceMap(logger, 0)
fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault} fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault}
u, err := url.Parse("/foo012") u, err := url.Parse("/foo012")
if err != nil { if err != nil {
router/httpTriggers.go
+19 -12
View File
@@ -18,11 +18,11 @@ package router
import ( import (
"context" "context"
"log"
"net/http" "net/http"
"time" "time"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/fields"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
@@ -39,6 +39,7 @@ type HTTPTriggerSet struct {
*functionServiceMap *functionServiceMap
*mutableRouter *mutableRouter
logger *zap.Logger
fissionClient *crd.FissionClient fissionClient *crd.FissionClient
kubeClient *kubernetes.Clientset kubeClient *kubernetes.Clientset
executor *executorClient.Client executor *executorClient.Client
@@ -57,10 +58,11 @@ type HTTPTriggerSet struct {
svcAddrUpdateThrottler *throttler.Throttler svcAddrUpdateThrottler *throttler.Throttler
} }
func makeHTTPTriggerSet(fmap *functionServiceMap, frmap *functionRecorderMap, trmap *triggerRecorderMap, fissionClient *crd.FissionClient, func makeHTTPTriggerSet(logger *zap.Logger, fmap *functionServiceMap, frmap *functionRecorderMap, trmap *triggerRecorderMap, fissionClient *crd.FissionClient,
kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient *rest.RESTClient, params *tsRoundTripperParams, isDebugEnv bool, actionThrottler *throttler.Throttler) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) { kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient *rest.RESTClient, params *tsRoundTripperParams, isDebugEnv bool, actionThrottler *throttler.Throttler) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
httpTriggerSet := &HTTPTriggerSet{ httpTriggerSet := &HTTPTriggerSet{
logger: logger.Named("http_trigger_set"),
functionServiceMap: fmap, functionServiceMap: fmap,
triggers: []crd.HTTPTrigger{}, triggers: []crd.HTTPTrigger{},
fissionClient: fissionClient, fissionClient: fissionClient,
@@ -83,7 +85,7 @@ func makeHTTPTriggerSet(fmap *functionServiceMap, frmap *functionRecorderMap, tr
httpTriggerSet.funcStore = fnStore httpTriggerSet.funcStore = fnStore
httpTriggerSet.funcController = fnController httpTriggerSet.funcController = fnController
} }
recorderSet = MakeRecorderSet(httpTriggerSet, crdClient, rStore, frmap, trmap) recorderSet = MakeRecorderSet(logger, httpTriggerSet, crdClient, rStore, frmap, trmap)
httpTriggerSet.recorderSet = recorderSet httpTriggerSet.recorderSet = recorderSet
return httpTriggerSet, tStore, fnStore return httpTriggerSet, tStore, fnStore
} }
@@ -95,7 +97,7 @@ func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter
if ts.fissionClient == nil { if ts.fissionClient == nil {
// Used in tests only. // Used in tests only.
log.Printf("Skipping continuous trigger updates") ts.logger.Info("skipping continuous trigger updates")
return return
} }
go ts.updateRouter() go ts.updateRouter()
@@ -104,7 +106,7 @@ func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter
if ts.recorderSet.recController != nil { if ts.recorderSet.recController != nil {
go ts.runWatcher(ctx, ts.recorderSet.recController) go ts.runWatcher(ctx, ts.recorderSet.recController)
} else { } else {
log.Fatal("Failed to run recorder Controller") ts.logger.Fatal("failed to run recorder controller")
} }
} }
@@ -143,10 +145,11 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
if rr.resolveResultType != resolveResultSingleFunction && rr.resolveResultType != resolveResultMultipleFunctions { if rr.resolveResultType != resolveResultSingleFunction && rr.resolveResultType != resolveResultMultipleFunctions {
// not implemented yet // not implemented yet
log.Panicf("resolve result type not implemented (%v)", rr.resolveResultType) ts.logger.Panic("resolve result type not implemented", zap.Any("type", rr.resolveResultType))
} }
fh := &functionHandler{ fh := &functionHandler{
logger: ts.logger.Named(trigger.Metadata.Name),
fmap: ts.functionServiceMap, fmap: ts.functionServiceMap,
frmap: ts.recorderSet.functionRecorderMap, frmap: ts.recorderSet.functionRecorderMap,
trmap: ts.recorderSet.triggerRecorderMap, trmap: ts.recorderSet.triggerRecorderMap,
@@ -205,6 +208,7 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
} }
fh := &functionHandler{ fh := &functionHandler{
logger: ts.logger.Named(m.Name),
fmap: ts.functionServiceMap, fmap: ts.functionServiceMap,
frmap: ts.recorderSet.functionRecorderMap, frmap: ts.recorderSet.functionRecorderMap,
trmap: ts.recorderSet.triggerRecorderMap, trmap: ts.recorderSet.triggerRecorderMap,
@@ -235,7 +239,7 @@ func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Cont
k8sCache.ResourceEventHandlerFuncs{ k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) { AddFunc: func(obj interface{}) {
trigger := obj.(*crd.HTTPTrigger) trigger := obj.(*crd.HTTPTrigger)
go createIngress(trigger, ts.kubeClient) go createIngress(ts.logger, trigger, ts.kubeClient)
ts.syncTriggers() ts.syncTriggers()
// Check if this trigger's function needs to be recorded // Check if this trigger's function needs to be recorded
fnRef := trigger.Spec.FunctionReference.Name fnRef := trigger.Spec.FunctionReference.Name
@@ -244,14 +248,17 @@ func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Cont
if len(recorder.Spec.Triggers) == 0 { if len(recorder.Spec.Triggers) == 0 {
ts.recorderSet.triggerRecorderMap.assign(trigger.Metadata.Name, recorder) ts.recorderSet.triggerRecorderMap.assign(trigger.Metadata.Name, recorder)
} }
} else if err != nil {
ts.logger.Error("unable to lookup function in functionRecorderMap", zap.Error(err))
} else { } else {
log.Print("Unable to lookup function in functionRecorderMap") ts.logger.Error("unable to lookup function in functionRecorderMap")
} }
}, },
DeleteFunc: func(obj interface{}) { DeleteFunc: func(obj interface{}) {
ts.syncTriggers() ts.syncTriggers()
trigger := obj.(*crd.HTTPTrigger) trigger := obj.(*crd.HTTPTrigger)
go deleteIngress(trigger, ts.kubeClient) go deleteIngress(ts.logger, trigger, ts.kubeClient)
go ts.recorderSet.DeleteTriggerFromRecorderMap(trigger) go ts.recorderSet.DeleteTriggerFromRecorderMap(trigger)
}, },
UpdateFunc: func(oldObj interface{}, newObj interface{}) { UpdateFunc: func(oldObj interface{}, newObj interface{}) {
@@ -262,7 +269,7 @@ func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Cont
return return
} }
go updateIngress(oldTrigger, newTrigger, ts.kubeClient) go updateIngress(ts.logger, oldTrigger, newTrigger, ts.kubeClient)
ts.syncTriggers() ts.syncTriggers()
}, },
}) })
@@ -296,10 +303,10 @@ func (ts *HTTPTriggerSet) initFunctionController() (k8sCache.Store, k8sCache.Con
rr.functionMetadataMap[fn.Metadata.Name] != nil && rr.functionMetadataMap[fn.Metadata.Name] != nil &&
rr.functionMetadataMap[fn.Metadata.Name].ResourceVersion != fn.Metadata.ResourceVersion { rr.functionMetadataMap[fn.Metadata.Name].ResourceVersion != fn.Metadata.ResourceVersion {
// invalidate resolver cache // invalidate resolver cache
log.Printf("Invalidating resolver cache") ts.logger.Info("invalidating resolver cache")
err := ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion) err := ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
if err != nil { if err != nil {
log.Printf("Error deleting functionReferenceResolver cache: %v", err) ts.logger.Error("error deleting functionReferenceResolver cache", zap.Error(err))
} }
break break
router/ingress.go
+20 -15
View File
@@ -17,9 +17,9 @@ limitations under the License.
package router package router
import ( import (
"log"
"os" "os"
"go.uber.org/zap"
"k8s.io/api/extensions/v1beta1" "k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -38,16 +38,16 @@ func init() {
} }
} }
func createIngress(trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { func createIngress(logger *zap.Logger, trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
if !trigger.Spec.CreateIngress { if !trigger.Spec.CreateIngress {
log.Printf("Skipping creation of ingress for trigger: %v", trigger.Metadata.Name) logger.Info("skipping creation of ingress for trigger", zap.String("trigger", trigger.Metadata.Name))
return return
} }
_, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{}) _, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{})
if err == nil { if err == nil {
log.Printf("Ingress for trigger exists already %v", trigger.Metadata.Name) logger.Info("ingress for trigger exists already", zap.String("trigger", trigger.Metadata.Name))
return return
} }
@@ -87,10 +87,10 @@ func createIngress(trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
_, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Create(ing) _, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Create(ing)
if err != nil { if err != nil {
log.Printf("Failed to create ingress: %v", err) logger.Error("failed to create ingress", zap.Error(err))
return return
} }
log.Printf("Created ingress successfully for trigger %v", trigger.Metadata.Name) logger.Info("created ingress successfully for trigger", zap.String("trigger", trigger.Metadata.Name))
} }
func getDeployLabels(trigger *crd.HTTPTrigger) map[string]string { func getDeployLabels(trigger *crd.HTTPTrigger) map[string]string {
@@ -101,41 +101,46 @@ func getDeployLabels(trigger *crd.HTTPTrigger) map[string]string {
} }
} }
func deleteIngress(trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { func deleteIngress(logger *zap.Logger, trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
if !trigger.Spec.CreateIngress { if !trigger.Spec.CreateIngress {
return return
} }
ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{}) ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{})
if err != nil { if err != nil {
log.Printf("Failed to get ingress when deleting trigger: %v, %v", err, trigger) logger.Error("failed to get ingress when deleting trigger", zap.Error(err), zap.String("trigger", trigger.Metadata.Name))
} }
err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Delete(ingress.Name, &v1.DeleteOptions{}) err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Delete(ingress.Name, &v1.DeleteOptions{})
if err != nil { if err != nil {
log.Printf("Failed to delete ingress %v error: %v", ingress, err) logger.Error("failed to delete ingress for trigger",
zap.Error(err),
zap.Any("ingress", ingress),
zap.String("trigger", trigger.Metadata.Name))
} }
} }
func updateIngress(oldT *crd.HTTPTrigger, newT *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) { func updateIngress(logger *zap.Logger, oldT *crd.HTTPTrigger, newT *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
if oldT.Spec.CreateIngress == false && newT.Spec.CreateIngress == true { if oldT.Spec.CreateIngress == false && newT.Spec.CreateIngress == true {
createIngress(newT, kubeClient) createIngress(logger, newT, kubeClient)
return return
} }
if newT.Spec.CreateIngress == false && oldT.Spec.CreateIngress == true { if newT.Spec.CreateIngress == false && oldT.Spec.CreateIngress == true {
deleteIngress(oldT, kubeClient) deleteIngress(logger, oldT, kubeClient)
return return
} }
if newT.Spec.Host != oldT.Spec.Host || newT.Spec.RelativeURL != oldT.Spec.RelativeURL { if newT.Spec.Host != oldT.Spec.Host || newT.Spec.RelativeURL != oldT.Spec.RelativeURL {
log.Printf("Updating ingress for trigger %v", oldT.Metadata.Name) logger.Info("updating ingress for trigger", zap.String("trigger", oldT.Metadata.Name))
ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(oldT.Metadata.Name, v1.GetOptions{}) ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(oldT.Metadata.Name, v1.GetOptions{})
if err != nil { if err != nil {
log.Printf("Failed to get ingress when updating trigger: %v", err) logger.Error("failed to get ingress when updating trigger",
zap.Error(err),
zap.String("trigger", oldT.Metadata.Name))
} }
if newT.Spec.Host != oldT.Spec.Host { if newT.Spec.Host != oldT.Spec.Host {
@@ -148,7 +153,7 @@ func updateIngress(oldT *crd.HTTPTrigger, newT *crd.HTTPTrigger, kubeClient *kub
_, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Update(ingress) _, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Update(ingress)
if err != nil { if err != nil {
log.Printf("Failed to update ingress for trigger: %v", err) logger.Error("failed to update ingress for trigger", zap.String("trigger", oldT.Metadata.Name))
} }
} }
router/mutablemux.go
+7 -4
View File
@@ -17,11 +17,11 @@ limitations under the License.
package router package router
import ( import (
"log"
"net/http" "net/http"
"sync/atomic" "sync/atomic"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"go.uber.org/zap"
) )
// //
@@ -30,11 +30,14 @@ import (
// //
type mutableRouter struct { type mutableRouter struct {
logger *zap.Logger
router atomic.Value // mux.Router router atomic.Value // mux.Router
} }
func NewMutableRouter(handler *mux.Router) *mutableRouter { func NewMutableRouter(logger *zap.Logger, handler *mux.Router) *mutableRouter {
mr := mutableRouter{} mr := mutableRouter{
logger: logger.Named("mutable_router"),
}
mr.router.Store(handler) mr.router.Store(handler)
return &mr return &mr
} }
@@ -44,7 +47,7 @@ func (mr *mutableRouter) ServeHTTP(responseWriter http.ResponseWriter, request *
routerValue := mr.router.Load() routerValue := mr.router.Load()
router, ok := routerValue.(*mux.Router) router, ok := routerValue.(*mux.Router)
if !ok { if !ok {
log.Panic("Invalid router type") mr.logger.Panic("invalid router type")
} }
router.ServeHTTP(responseWriter, request) router.ServeHTTP(responseWriter, request)
} }
router/mutablemux_test.go
+7 -2
View File
@@ -17,11 +17,13 @@ limitations under the License.
package router package router
import ( import (
"github.com/gorilla/mux"
"log" "log"
"net/http" "net/http"
"testing" "testing"
"time" "time"
"github.com/gorilla/mux"
"go.uber.org/zap"
) )
func OldHandler(responseWriter http.ResponseWriter, request *http.Request) { func OldHandler(responseWriter http.ResponseWriter, request *http.Request) {
@@ -62,7 +64,10 @@ func TestMutableMux(t *testing.T) {
log.Print("Create mutable router") log.Print("Create mutable router")
muxRouter := mux.NewRouter() muxRouter := mux.NewRouter()
muxRouter.HandleFunc("/", OldHandler) muxRouter.HandleFunc("/", OldHandler)
mr := NewMutableRouter(muxRouter) logger, err := zap.NewDevelopment()
panicIf(err)
mr := NewMutableRouter(logger, muxRouter)
// start http server // start http server
log.Print("Start http server") log.Print("Start http server")
router/recorderController.go
+26 -9
View File
@@ -1,13 +1,16 @@
package router package router
import ( import (
"github.com/fission/fission/crd" "go.uber.org/zap"
log "github.com/sirupsen/logrus"
"k8s.io/client-go/rest" "k8s.io/client-go/rest"
k8sCache "k8s.io/client-go/tools/cache" k8sCache "k8s.io/client-go/tools/cache"
"github.com/fission/fission/crd"
) )
type RecorderSet struct { type RecorderSet struct {
logger *zap.Logger
httpTriggerSet *HTTPTriggerSet httpTriggerSet *HTTPTriggerSet
crdClient *rest.RESTClient crdClient *rest.RESTClient
@@ -19,8 +22,9 @@ type RecorderSet struct {
triggerRecorderMap *triggerRecorderMap triggerRecorderMap *triggerRecorderMap
} }
func MakeRecorderSet(httpTriggerSet *HTTPTriggerSet, crdClient *rest.RESTClient, rStore k8sCache.Store, frmap *functionRecorderMap, trmap *triggerRecorderMap) *RecorderSet { func MakeRecorderSet(logger *zap.Logger, httpTriggerSet *HTTPTriggerSet, crdClient *rest.RESTClient, rStore k8sCache.Store, frmap *functionRecorderMap, trmap *triggerRecorderMap) *RecorderSet {
recorderSet := &RecorderSet{ recorderSet := &RecorderSet{
logger: logger.Named("recorder_set"),
httpTriggerSet: httpTriggerSet, httpTriggerSet: httpTriggerSet,
crdClient: crdClient, crdClient: crdClient,
recStore: rStore, recStore: rStore,
@@ -63,12 +67,17 @@ func (rs *RecorderSet) disableRecorder(r *crd.Recorder) {
function := r.Spec.Function function := r.Spec.Function
triggers := r.Spec.Triggers triggers := r.Spec.Triggers
log.Info("Disabling recorder ", r.Metadata.Name) rs.logger.Info("disabling recorder",
zap.String("recorder", r.Metadata.Name),
zap.String("function", function))
// Account for function // Account for function
err := rs.functionRecorderMap.remove(function) err := rs.functionRecorderMap.remove(function)
if err != nil { if err != nil {
log.Error("Error disabling recorder (failed to remove function from functionRecorderMap): ", err) rs.logger.Error("error disabling recorder (failed to remove function from functionRecorderMap)",
zap.Error(err),
zap.String("recorder", r.Metadata.Name),
zap.String("function", function))
} }
// Account for explicitly added triggers // Account for explicitly added triggers
@@ -76,7 +85,11 @@ func (rs *RecorderSet) disableRecorder(r *crd.Recorder) {
for _, trigger := range triggers { for _, trigger := range triggers {
err := rs.triggerRecorderMap.remove(trigger) err := rs.triggerRecorderMap.remove(trigger)
if err != nil { if err != nil {
log.Error("Error disabling recorder (failed to remove triggers from triggerRecorderMap): ", err) rs.logger.Error("error disabling recorder (failed to remove triggers from triggerRecorderMap)",
zap.Error(err),
zap.String("recorder", r.Metadata.Name),
zap.String("function", function),
zap.String("trigger", trigger))
} }
} }
} else { } else {
@@ -86,7 +99,11 @@ func (rs *RecorderSet) disableRecorder(r *crd.Recorder) {
if trigger.Spec.FunctionReference.Name == function { if trigger.Spec.FunctionReference.Name == function {
err := rs.triggerRecorderMap.remove(trigger.Metadata.Name) err := rs.triggerRecorderMap.remove(trigger.Metadata.Name)
if err != nil { if err != nil {
log.Error("Failed to remove trigger from triggerRecorderMap: ", err) rs.logger.Error("failed to remove trigger from triggerRecorderMap",
zap.Error(err),
zap.String("recorder", r.Metadata.Name),
zap.String("function", function),
zap.String("trigger", trigger.Metadata.Name))
} }
} }
} }
@@ -106,13 +123,13 @@ func (rs *RecorderSet) updateRecorder(old *crd.Recorder, newer *crd.Recorder) {
func (rs *RecorderSet) DeleteTriggerFromRecorderMap(trigger *crd.HTTPTrigger) { func (rs *RecorderSet) DeleteTriggerFromRecorderMap(trigger *crd.HTTPTrigger) {
err := rs.triggerRecorderMap.remove(trigger.Metadata.Name) err := rs.triggerRecorderMap.remove(trigger.Metadata.Name)
if err != nil { if err != nil {
log.Error("Failed to remove trigger from triggerRecorderMap: ", err) rs.logger.Error("failed to remove trigger from triggerRecorderMap", zap.Error(err))
} }
} }
func (rs *RecorderSet) DeleteFunctionFromRecorderMap(function *crd.Function) { func (rs *RecorderSet) DeleteFunctionFromRecorderMap(function *crd.Function) {
err := rs.functionRecorderMap.remove(function.Metadata.Name) err := rs.functionRecorderMap.remove(function.Metadata.Name)
if err != nil { if err != nil {
log.Error("Failed to remove function from functionRecorderMap: ", err) rs.logger.Error("failed to remove function from functionRecorderMap", zap.Error(err))
} }
} }
router/router.go
+57 -32
View File
@@ -42,7 +42,6 @@ package router
import ( import (
"context" "context"
"fmt" "fmt"
"log"
"net/http" "net/http"
"os" "os"
"strconv" "strconv"
@@ -52,6 +51,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"go.opencensus.io/plugin/ochttp" "go.opencensus.io/plugin/ochttp"
"go.opencensus.io/trace" "go.opencensus.io/trace"
"go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
@@ -63,16 +63,16 @@ import (
// request url ---[trigger]---> Function(name, deployment) ----[deployment]----> Function(name, uid) ----[pool mgr]---> k8s service url // request url ---[trigger]---> Function(name, deployment) ----[deployment]----> Function(name, uid) ----[pool mgr]---> k8s service url
func router(ctx context.Context, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) *mutableRouter { func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) *mutableRouter {
muxRouter := mux.NewRouter() muxRouter := mux.NewRouter()
mr := NewMutableRouter(muxRouter) mr := NewMutableRouter(logger, muxRouter)
muxRouter.Use(fission.LoggingMiddleware) muxRouter.Use(fission.LoggingMiddleware(logger))
httpTriggerSet.subscribeRouter(ctx, mr, resolver) httpTriggerSet.subscribeRouter(ctx, mr, resolver)
return mr return mr
} }
func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) { func serve(ctx context.Context, logger *zap.Logger, port int, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) {
mr := router(ctx, httpTriggerSet, resolver) mr := router(ctx, logger, httpTriggerSet, resolver)
url := fmt.Sprintf(":%v", port) url := fmt.Sprintf(":%v", port)
http.ListenAndServe(url, &ochttp.Handler{ http.ListenAndServe(url, &ochttp.Handler{
Handler: mr, Handler: mr,
@@ -82,77 +82,102 @@ func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolv
}) })
} }
func serveMetric() { func serveMetric(logger *zap.Logger) {
// Expose the registered metrics via HTTP. // Expose the registered metrics via HTTP.
http.Handle("/metrics", promhttp.Handler()) http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(metricAddr, nil)) err := http.ListenAndServe(metricAddr, nil)
logger.Fatal("done listening on metrics endpoint", zap.Error(err))
} }
func Start(port int, executorUrl string) { func Start(logger *zap.Logger, port int, executorUrl string) {
// setup a signal handler for SIGTERM // setup a signal handler for SIGTERM
fission.SetupStackTraceHandler() fission.SetupStackTraceHandler()
fmap := makeFunctionServiceMap(time.Minute) fmap := makeFunctionServiceMap(logger, time.Minute)
frmap := makeFunctionRecorderMap(time.Minute) frmap := makeFunctionRecorderMap(logger, time.Minute)
trmap := makeTriggerRecorderMap(time.Minute) trmap := makeTriggerRecorderMap(logger, time.Minute)
fissionClient, kubeClient, _, err := crd.MakeFissionClient() fissionClient, kubeClient, _, err := crd.MakeFissionClient()
if err != nil { if err != nil {
log.Fatalf("Error connecting to kubernetes API: %v", err) logger.Fatal("error connecting to kubernetes API", zap.Error(err))
} }
err = fissionClient.WaitForCRDs() err = fissionClient.WaitForCRDs()
if err != nil { if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err) logger.Fatal("error waiting for CRDs", zap.Error(err))
} }
restClient := fissionClient.GetCrdClient() restClient := fissionClient.GetCrdClient()
executor := executorClient.MakeClient(executorUrl) executor := executorClient.MakeClient(logger, executorUrl)
timeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_TIMEOUT")) timeoutStr := os.Getenv("ROUTER_ROUND_TRIP_TIMEOUT")
timeout, err := time.ParseDuration(timeoutStr)
if err != nil { if err != nil {
log.Fatalf("Failed to parse timeout: %v", err) logger.Fatal("failed to parse timeout duration from 'ROUTER_ROUND_TRIP_TIMEOUT'",
zap.Error(err),
zap.String("value", timeoutStr))
} }
timeoutExponent, err := strconv.Atoi(os.Getenv("ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT")) timeoutExponentStr := os.Getenv("ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT")
timeoutExponent, err := strconv.Atoi(timeoutExponentStr)
if err != nil { if err != nil {
log.Fatalf("Failed to parse timeout exponent: %v", err) logger.Fatal("failed to parse timeout exponent from 'ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT'",
zap.Error(err),
zap.String("value", timeoutExponentStr))
} }
keepAlive, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME")) keepAliveStr := os.Getenv("ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME")
keepAlive, err := time.ParseDuration(keepAliveStr)
if err != nil { if err != nil {
log.Fatalf("Failed to parse keep alive time: %v", err) logger.Fatal("failed to parse keep alive duration from 'ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME'",
zap.Error(err),
zap.String("value", keepAliveStr))
} }
maxRetries, err := strconv.Atoi(os.Getenv("ROUTER_ROUND_TRIP_MAX_RETRIES")) maxRetriesStr := os.Getenv("ROUTER_ROUND_TRIP_MAX_RETRIES")
maxRetries, err := strconv.Atoi(maxRetriesStr)
if err != nil { if err != nil {
log.Fatalf("Failed to parse max retry times: %v", err) logger.Fatal("failed to parse max retries from 'ROUTER_ROUND_TRIP_MAX_RETRIES'",
zap.Error(err),
zap.String("value", maxRetriesStr))
} }
isDebugEnv, err := strconv.ParseBool(os.Getenv("DEBUG_ENV")) isDebugEnvStr := os.Getenv("DEBUG_ENV")
isDebugEnv, err := strconv.ParseBool(isDebugEnvStr)
if err != nil { if err != nil {
log.Fatalf("Failed to parse DEBUG_ENV: %v", err) logger.Fatal("failed to parse debug env from 'DEBUG_ENV'",
zap.Error(err),
zap.String("value", isDebugEnvStr))
} }
// svcAddrRetryCount is the max times for RetryingRoundTripper to retry with a specific service address // svcAddrRetryCount is the max times for RetryingRoundTripper to retry with a specific service address
svcAddrRetryCount, err := strconv.Atoi(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES")) svcAddrRetryCountStr := os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES")
svcAddrRetryCount, err := strconv.Atoi(svcAddrRetryCountStr)
if err != nil { if err != nil {
svcAddrRetryCount = 5 svcAddrRetryCount = 5
log.Printf("Failed to parse svc address retry conunt, set it to default value(5): %v", err) logger.Info("failed to parse service address retry count from 'ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES' - set to the default value",
zap.Error(err),
zap.String("value", svcAddrRetryCountStr),
zap.Int("default", svcAddrRetryCount))
} }
// svcAddrUpdateTimeout is the timeout setting for a goroutine to wait for the update of a service entry. // svcAddrUpdateTimeout is the timeout setting for a goroutine to wait for the update of a service entry.
// If the update process cannot be done within the timeout window, consider it failed. // If the update process cannot be done within the timeout window, consider it failed.
svcAddrUpdateTimeoutStr := os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT")
svcAddrUpdateTimeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT")) svcAddrUpdateTimeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT"))
if err != nil { if err != nil {
svcAddrUpdateTimeout = 30 * time.Second svcAddrUpdateTimeout = 30 * time.Second
log.Printf("Failed to parse svc address update timeout, set it to default value(30): %v", err) logger.Info("failed to parse service address update timeout duration from 'ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT' - set to the default value",
zap.Error(err),
zap.String("value", svcAddrUpdateTimeoutStr),
zap.Duration("default", svcAddrUpdateTimeout))
} }
triggers, _, fnStore := makeHTTPTriggerSet(fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{ triggers, _, fnStore := makeHTTPTriggerSet(logger.Named("triggerset"), fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{
timeout: timeout, timeout: timeout,
timeoutExponent: timeoutExponent, timeoutExponent: timeoutExponent,
keepAlive: keepAlive, keepAlive: keepAlive,
@@ -162,10 +187,10 @@ func Start(port int, executorUrl string) {
resolver := makeFunctionReferenceResolver(fnStore) resolver := makeFunctionReferenceResolver(fnStore)
go serveMetric() go serveMetric(logger)
log.Printf("Starting router at port %v\n", port) logger.Info("starting router", zap.Int("port", port))
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
serve(ctx, port, triggers, resolver) serve(ctx, logger, port, triggers, resolver)
} }
router/router_test.go
+9 -5
View File
@@ -22,6 +22,7 @@ import (
"testing" "testing"
"time" "time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
@@ -43,16 +44,19 @@ func TestRouter(t *testing.T) {
testResponseString := "hi" testResponseString := "hi"
testServiceUrl := createBackendService(testResponseString) testServiceUrl := createBackendService(testResponseString)
logger, err := zap.NewDevelopment()
panicIf(err)
// set up the cache with this fake service // set up the cache with this fake service
fmap := makeFunctionServiceMap(0) fmap := makeFunctionServiceMap(logger, 0)
fmap.assign(fn, testServiceUrl) fmap.assign(fn, testServiceUrl)
frmap := makeFunctionRecorderMap(time.Minute) frmap := makeFunctionRecorderMap(logger, time.Minute)
trmap := makeTriggerRecorderMap(time.Minute) trmap := makeTriggerRecorderMap(logger, time.Minute)
// HTTP trigger set with a trigger for this function // HTTP trigger set with a trigger for this function
triggers, _, _ := makeHTTPTriggerSet(fmap, frmap, trmap, nil, nil, nil, nil, triggers, _, _ := makeHTTPTriggerSet(logger, fmap, frmap, trmap, nil, nil, nil, nil,
&tsRoundTripperParams{ &tsRoundTripperParams{
timeout: 50 * time.Millisecond, timeout: 50 * time.Millisecond,
timeoutExponent: 2, timeoutExponent: 2,
@@ -95,7 +99,7 @@ func TestRouter(t *testing.T) {
port := 4242 port := 4242
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
go serve(ctx, port, triggers, frr) go serve(ctx, logger, port, triggers, frr)
time.Sleep(100 * time.Millisecond) time.Sleep(100 * time.Millisecond)
// hit the router // hit the router
router/triggerRecorderMap.go
+8 -5
View File
@@ -17,9 +17,10 @@ limitations under the License.
package router package router
import ( import (
"log"
"time" "time"
"go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/cache" "github.com/fission/fission/cache"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
@@ -27,13 +28,15 @@ import (
type ( type (
triggerRecorderMap struct { triggerRecorderMap struct {
cache *cache.Cache // map[string]*crd.Recorder logger *zap.Logger
cache *cache.Cache // map[string]*crd.Recorder
} }
) )
func makeTriggerRecorderMap(expiry time.Duration) *triggerRecorderMap { func makeTriggerRecorderMap(logger *zap.Logger, expiry time.Duration) *triggerRecorderMap {
return &triggerRecorderMap{ return &triggerRecorderMap{
cache: cache.MakeCache(expiry, 0), logger: logger.Named("trigger_recorder_map"),
cache: cache.MakeCache(expiry, 0),
} }
} }
@@ -52,7 +55,7 @@ func (trmap *triggerRecorderMap) assign(trigger string, recorder *crd.Recorder)
if e, ok := err.(fission.Error); ok && e.Code == fission.ErrorNameExists { if e, ok := err.(fission.Error); ok && e.Code == fission.ErrorNameExists {
return return
} }
log.Printf("error caching recorder for function name with a different value: %v", err) trmap.logger.Error("error caching recorder for function name with a different value", zap.Error(err))
} }
} }
router/util_test.go
+6
View File
@@ -28,3 +28,9 @@ func testRequest(targetUrl string, expectedResponse string) {
log.Panic("Unexpected response") log.Panic("Unexpected response")
} }
} }
func panicIf(err error) {
if err != nil {
log.Panicf("Error: %v", err)
}
}
storagesvc/archivePruner.go
+29 -20
View File
@@ -19,13 +19,14 @@ package storagesvc
import ( import (
"time" "time"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
) )
type ArchivePruner struct { type ArchivePruner struct {
logger *zap.Logger
crdClient *crd.FissionClient crdClient *crd.FissionClient
archiveChan chan (string) archiveChan chan (string)
stowClient *StowClient stowClient *StowClient
@@ -34,13 +35,14 @@ type ArchivePruner struct {
const defaultPruneInterval int = 60 // in minutes const defaultPruneInterval int = 60 // in minutes
func MakeArchivePruner(stowClient *StowClient, pruneInterval time.Duration) (*ArchivePruner, error) { func MakeArchivePruner(logger *zap.Logger, stowClient *StowClient, pruneInterval time.Duration) (*ArchivePruner, error) {
crdClient, _, _, err := crd.MakeFissionClient() crdClient, _, _, err := crd.MakeFissionClient()
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &ArchivePruner{ return &ArchivePruner{
logger: logger.Named("archive_pruner"),
crdClient: crdClient, crdClient: crdClient,
archiveChan: make(chan string), archiveChan: make(chan string),
stowClient: stowClient, stowClient: stowClient,
@@ -50,15 +52,18 @@ func MakeArchivePruner(stowClient *StowClient, pruneInterval time.Duration) (*Ar
// pruneArchives listens to archiveChannel for archive ids that need to be deleted // pruneArchives listens to archiveChannel for archive ids that need to be deleted
func (pruner *ArchivePruner) pruneArchives() { func (pruner *ArchivePruner) pruneArchives() {
log.Info("listening to archiveChannel to prune archives") pruner.logger.Info("listening to archiveChannel to prune archives")
for { for {
select { select {
case archiveID := <-pruner.archiveChan: case archiveID := <-pruner.archiveChan:
log.WithField("archive ID", archiveID).Info("sending delete request") pruner.logger.Info("sending delete request for archive",
zap.String("archive_id", archiveID))
if err := pruner.stowClient.removeFileByID(archiveID); err != nil { if err := pruner.stowClient.removeFileByID(archiveID); err != nil {
// logging the error and continuing with other deletions. // logging the error and continuing with other deletions.
// hopefully this archive will be deleted in the next iteration. // hopefully this archive will be deleted in the next iteration.
log.WithField("archiveID", archiveID).WithError(err).Error("Ignoring error while deleting archive") pruner.logger.Error("ignoring error while deleting archive",
zap.Error(err),
zap.String("archive_id", archiveID))
} }
} }
} }
@@ -72,16 +77,16 @@ func (pruner *ArchivePruner) insertArchive(archiveID string) {
// A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes // A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes
// and not the archives that are referenced by them, leaving the archives as orphans. // and not the archives that are referenced by them, leaving the archives as orphans.
// getOrphanArchives reaps the orphaned archives. // getOrphanArchives reaps the orphaned archives.
func (pruner *ArchivePruner) getOrphanArchives() error { func (pruner *ArchivePruner) getOrphanArchives() {
log.Info("getting orphan archives") pruner.logger.Info("getting orphan archives")
archivesRefByPkgs := make([]string, 0) archivesRefByPkgs := make([]string, 0)
var archiveID string var archiveID string
// get all pkgs from kubernetes // get all pkgs from kubernetes
pkgList, err := pruner.crdClient.Packages(metav1.NamespaceAll).List(metav1.ListOptions{}) pkgList, err := pruner.crdClient.Packages(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil { if err != nil {
log.WithError(err).Error("Error getting package list from kubernetes") pruner.logger.Error("error getting package list from kubernetes", zap.Error(err))
return err return
} }
// extract archives referenced by these pkgs // extract archives referenced by these pkgs
@@ -89,44 +94,48 @@ func (pruner *ArchivePruner) getOrphanArchives() error {
if pkg.Spec.Deployment.URL != "" { if pkg.Spec.Deployment.URL != "" {
archiveID, err = getQueryParamValue(pkg.Spec.Deployment.URL, "id") archiveID, err = getQueryParamValue(pkg.Spec.Deployment.URL, "id")
if err != nil { if err != nil {
log.WithError(err).Error("Error extracting value of archiveID from url") pruner.logger.Error("error extracting value of archiveID from deployment url",
return err zap.Error(err),
zap.String("url", pkg.Spec.Deployment.URL))
return
} }
archivesRefByPkgs = append(archivesRefByPkgs, archiveID) archivesRefByPkgs = append(archivesRefByPkgs, archiveID)
} }
if pkg.Spec.Source.URL != "" { if pkg.Spec.Source.URL != "" {
archiveID, err = getQueryParamValue(pkg.Spec.Source.URL, "id") archiveID, err = getQueryParamValue(pkg.Spec.Source.URL, "id")
if err != nil { if err != nil {
log.WithError(err).Error("Error extracting value of archiveID from url") pruner.logger.Error("error extracting value of archiveID from source url",
return err zap.Error(err),
zap.String("url", pkg.Spec.Source.URL))
return
} }
archivesRefByPkgs = append(archivesRefByPkgs, archiveID) archivesRefByPkgs = append(archivesRefByPkgs, archiveID)
} }
} }
log.WithField("list", "archives referenced by packages").Debugf("%s", archivesRefByPkgs) pruner.logger.Debug("archives referenced by packagese", zap.Strings("archives", archivesRefByPkgs))
// get all archives on storage // get all archives on storage
// out of them, there may be some just created but not referenced by packages yet. // out of them, there may be some just created but not referenced by packages yet.
// need to filter them out. // need to filter them out.
archivesInStorage, err := pruner.stowClient.getItemIDsWithFilter(filterItemCreatedAMinuteAgo, time.Now()) archivesInStorage, err := pruner.stowClient.getItemIDsWithFilter(pruner.stowClient.filterItemCreatedAMinuteAgo, time.Now())
if err != nil { if err != nil {
log.WithError(err).Error("Error getting items from storage") pruner.logger.Error("error getting items from storage", zap.Error(err))
return err return
} }
log.WithField("list", "archives in storage").Debugf("%s", archivesInStorage) pruner.logger.Debug("archives in storage", zap.Strings("archives", archivesInStorage))
// difference of the two lists gives us the list of orphan archives. This is just a brute force approach. // difference of the two lists gives us the list of orphan archives. This is just a brute force approach.
// need to do something more optimal at scale. // need to do something more optimal at scale.
orphanedArchives := getDifferenceOfLists(archivesInStorage, archivesRefByPkgs) orphanedArchives := getDifferenceOfLists(archivesInStorage, archivesRefByPkgs)
log.WithField("list", "orphan archives").Debugf("%s", orphanedArchives) pruner.logger.Debug("orphan archives", zap.Strings("archives", orphanedArchives))
// send each orphan archive away for deletion // send each orphan archive away for deletion
for _, archiveID = range orphanedArchives { for _, archiveID = range orphanedArchives {
pruner.insertArchive(archiveID) pruner.insertArchive(archiveID)
} }
return nil return
} }
// Start starts a go routine that listens to a channel for archive IDs that need to deleted. // Start starts a go routine that listens to a channel for archive IDs that need to deleted.
storagesvc/client/storagesvc_test.go
+5 -1
View File
@@ -27,6 +27,7 @@ import (
"time" "time"
"github.com/dchest/uniuri" "github.com/dchest/uniuri"
"go.uber.org/zap"
"github.com/fission/fission/storagesvc" "github.com/fission/fission/storagesvc"
) )
@@ -52,9 +53,12 @@ func TestStorageService(t *testing.T) {
port := 8080 port := 8080
enableArchivePruner := false enableArchivePruner := false
logger, err := zap.NewDevelopment()
panicIf(err)
log.Println("starting storage svc") log.Println("starting storage svc")
_ = storagesvc.RunStorageService( _ = storagesvc.RunStorageService(
storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner) logger, storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner)
time.Sleep(time.Second) time.Sleep(time.Second)
client := MakeClient(fmt.Sprintf("http://localhost:%v/", port)) client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))
storagesvc/storagesvc.go
+30 -20
View File
@@ -25,15 +25,17 @@ import (
"strconv" "strconv"
"time" "time"
"github.com/fission/fission"
"github.com/gorilla/mux" "github.com/gorilla/mux"
_ "github.com/graymeta/stow/local" _ "github.com/graymeta/stow/local"
log "github.com/sirupsen/logrus"
"go.opencensus.io/plugin/ochttp" "go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
"github.com/fission/fission"
) )
type ( type (
StorageService struct { StorageService struct {
logger *zap.Logger
storageClient *StowClient storageClient *StowClient
port int port int
} }
@@ -61,27 +63,34 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request)
fileSizeS, ok := r.Header["X-File-Size"] fileSizeS, ok := r.Header["X-File-Size"]
if !ok { if !ok {
log.Error("Missing X-File-Size") ss.logger.Error("upload is missing the 'X-File-Size' header",
zap.String("filename", handler.Filename))
http.Error(w, "missing X-File-Size header", http.StatusBadRequest) http.Error(w, "missing X-File-Size header", http.StatusBadRequest)
return return
} }
fileSize, err := strconv.Atoi(fileSizeS[0]) fileSize, err := strconv.Atoi(fileSizeS[0])
if err != nil { if err != nil {
log.WithError(err).Errorf("Error parsing x-file-size: '%v'", fileSizeS) ss.logger.Error("error parsing 'X-File-Size' header",
zap.Error(err),
zap.Strings("header", fileSizeS),
zap.String("filename", handler.Filename))
http.Error(w, "missing or bad X-File-Size header", http.StatusBadRequest) http.Error(w, "missing or bad X-File-Size header", http.StatusBadRequest)
return return
} }
// TODO: allow headers to add more metadata (e.g. environment // TODO: allow headers to add more metadata (e.g. environment
// and function metadata) // and function metadata)
log.Infof("Handling upload for %v", handler.Filename) ss.logger.Info("handling upload",
zap.String("filename", handler.Filename))
//fileMetadata := make(map[string]interface{}) //fileMetadata := make(map[string]interface{})
//fileMetadata["filename"] = handler.Filename //fileMetadata["filename"] = handler.Filename
id, err := ss.storageClient.putFile(file, int64(fileSize)) id, err := ss.storageClient.putFile(file, int64(fileSize))
if err != nil { if err != nil {
log.WithError(err).Error("Error saving uploaded file") ss.logger.Error("error saving uploaded file",
zap.Error(err),
zap.String("filename", handler.Filename))
http.Error(w, "Error saving uploaded file", http.StatusInternalServerError) http.Error(w, "Error saving uploaded file", http.StatusInternalServerError)
return return
} }
@@ -92,6 +101,9 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request)
} }
resp, err := json.Marshal(ur) resp, err := json.Marshal(ur)
if err != nil { if err != nil {
ss.logger.Error("error marshaling uploaded file response",
zap.Error(err),
zap.String("filename", handler.Filename))
http.Error(w, "Error marshaling response", http.StatusInternalServerError) http.Error(w, "Error marshaling response", http.StatusInternalServerError)
return return
} }
@@ -136,7 +148,7 @@ func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request
// stream it to response // stream it to response
err = ss.storageClient.copyFileToStream(fileId, w) err = ss.storageClient.copyFileToStream(fileId, w)
if err != nil { if err != nil {
log.WithError(err).Errorf("Error getting item id '%v'", fileId) ss.logger.Error("error getting file from storage client", zap.Error(err), zap.String("file_id", fileId))
if err == ErrNotFound { if err == ErrNotFound {
http.Error(w, "Error retrieving item: not found", http.StatusNotFound) http.Error(w, "Error retrieving item: not found", http.StatusNotFound)
} else if err == ErrRetrievingItem { } else if err == ErrRetrievingItem {
@@ -154,8 +166,9 @@ func (ss *StorageService) healthHandler(w http.ResponseWriter, r *http.Request)
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
} }
func MakeStorageService(storageClient *StowClient, port int) *StorageService { func MakeStorageService(logger *zap.Logger, storageClient *StowClient, port int) *StorageService {
return &StorageService{ return &StorageService{
logger: logger.Named("storage_service"),
storageClient: storageClient, storageClient: storageClient,
port: port, port: port,
} }
@@ -170,30 +183,27 @@ func (ss *StorageService) Start(port int) {
address := fmt.Sprintf(":%v", port) address := fmt.Sprintf(":%v", port)
r.Use(fission.LoggingMiddleware) r.Use(fission.LoggingMiddleware(ss.logger))
err := http.ListenAndServe(address, &ochttp.Handler{ err := http.ListenAndServe(address, &ochttp.Handler{
Handler: r, Handler: r,
// Propagation: &b3.HTTPFormat{}, // Propagation: &b3.HTTPFormat{},
}) })
log.Fatal(err) ss.logger.Fatal("done listening", zap.Error(err))
} }
func RunStorageService(storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService { func RunStorageService(logger *zap.Logger, storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService {
// setup a signal handler for SIGTERM // setup a signal handler for SIGTERM
fission.SetupStackTraceHandler() fission.SetupStackTraceHandler()
// initialize logger
log.SetLevel(log.InfoLevel)
// create a storage client // create a storage client
storageClient, err := MakeStowClient(storageType, storagePath, containerName) storageClient, err := MakeStowClient(logger, storageType, storagePath, containerName)
if err != nil { if err != nil {
log.Fatalf("Error creating stowClient: %v", err) logger.Fatal("error creating stowClient", zap.Error(err))
} }
// create http handlers // create http handlers
storageService := MakeStorageService(storageClient, port) storageService := MakeStorageService(logger, storageClient, port)
go storageService.Start(port) go storageService.Start(port)
// enablePruner prevents storagesvc unit test from needing to talk to kubernetes // enablePruner prevents storagesvc unit test from needing to talk to kubernetes
@@ -203,13 +213,13 @@ func RunStorageService(storageType StorageType, storagePath string, containerNam
if err != nil { if err != nil {
pruneInterval = defaultPruneInterval pruneInterval = defaultPruneInterval
} }
pruner, err := MakeArchivePruner(storageClient, time.Duration(pruneInterval)) pruner, err := MakeArchivePruner(logger, storageClient, time.Duration(pruneInterval))
if err != nil { if err != nil {
log.Fatalf("Error creating archivePruner: %v", err) logger.Fatal("error creating archivePruner", zap.Error(err))
} }
go pruner.Start() go pruner.Start()
} }
log.Info("Storage service started") logger.Info("storage service started")
return storageService return storageService
} }
storagesvc/stowClient.go
+16 -11
View File
@@ -17,7 +17,6 @@ limitations under the License.
package storagesvc package storagesvc
import ( import (
"errors"
"io" "io"
"mime/multipart" "mime/multipart"
"os" "os"
@@ -25,8 +24,9 @@ import (
"github.com/graymeta/stow" "github.com/graymeta/stow"
_ "github.com/graymeta/stow/local" _ "github.com/graymeta/stow/local"
"github.com/pkg/errors"
"github.com/satori/go.uuid" "github.com/satori/go.uuid"
log "github.com/sirupsen/logrus" "go.uber.org/zap"
) )
type ( type (
@@ -40,6 +40,7 @@ type (
} }
StowClient struct { StowClient struct {
logger *zap.Logger
config *storageConfig config *storageConfig
location stow.Location location stow.Location
container stow.Container container stow.Container
@@ -59,7 +60,7 @@ var (
ErrWritingFileIntoResponse = errors.New("unable to copy item into http response") ErrWritingFileIntoResponse = errors.New("unable to copy item into http response")
) )
func MakeStowClient(storageType StorageType, storagePath string, containerName string) (*StowClient, error) { func MakeStowClient(logger *zap.Logger, storageType StorageType, storagePath string, containerName string) (*StowClient, error) {
if storageType != StorageTypeLocal { if storageType != StorageTypeLocal {
return nil, errors.New("Storage types other than 'local' are not implemented") return nil, errors.New("Storage types other than 'local' are not implemented")
} }
@@ -71,13 +72,13 @@ func MakeStowClient(storageType StorageType, storagePath string, containerName s
} }
stowClient := &StowClient{ stowClient := &StowClient{
logger: logger.Named("stow_client"),
config: config, config: config,
} }
cfg := stow.ConfigMap{"path": config.localPath} cfg := stow.ConfigMap{"path": config.localPath}
loc, err := stow.Dial("local", cfg) loc, err := stow.Dial("local", cfg)
if err != nil { if err != nil {
log.WithError(err).Error("Error initializing storage")
return nil, err return nil, err
} }
stowClient.location = loc stowClient.location = loc
@@ -99,7 +100,6 @@ func MakeStowClient(storageType StorageType, storagePath string, containerName s
} }
} }
if err != nil { if err != nil {
log.WithError(err).Error("Error initializing storage")
return nil, err return nil, err
} }
stowClient.container = con stowClient.container = con
@@ -116,11 +116,13 @@ func (client *StowClient) putFile(file multipart.File, fileSize int64) (string,
// save the file to the storage backend // save the file to the storage backend
item, err := client.container.Put(uploadName, file, int64(fileSize), nil) item, err := client.container.Put(uploadName, file, int64(fileSize), nil)
if err != nil { if err != nil {
log.WithError(err).Errorf("Error writing file: %s on storage", uploadName) client.logger.Error("error writing file on storage",
zap.Error(err),
zap.String("file", uploadName))
return "", ErrWritingFile return "", ErrWritingFile
} }
log.Debugf("Successfully wrote file:%s on storage", uploadName) client.logger.Debug("successfully wrote file on storage", zap.String("file", uploadName))
return item.ID(), nil return item.ID(), nil
} }
@@ -146,7 +148,7 @@ func (client *StowClient) copyFileToStream(fileId string, w io.Writer) error {
return ErrWritingFileIntoResponse return ErrWritingFileIntoResponse
} }
log.Debugf("successfully wrote file: %s into httpresponse", fileId) client.logger.Debug("successfully wrote file into httpresponse", zap.String("file", fileId))
return nil return nil
} }
@@ -169,7 +171,7 @@ func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncPara
for { for {
items, cursor, err = client.container.Items(stow.NoPrefix, cursor, PaginationSize) items, cursor, err = client.container.Items(stow.NoPrefix, cursor, PaginationSize)
if err != nil { if err != nil {
log.WithError(err).Error("Error getting items from container") errors.Wrap(err, "error getting items from container")
return nil, err return nil, err
} }
@@ -191,10 +193,13 @@ func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncPara
// filterItemCreatedAMinuteAgo is one type of filter function that filters out items created less than a minute ago. // filterItemCreatedAMinuteAgo is one type of filter function that filters out items created less than a minute ago.
// More filter functions can be written if needed, as long as they are of type filter // More filter functions can be written if needed, as long as they are of type filter
func filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool { func (client StowClient) filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool {
itemLastModTime, _ := item.LastMod() itemLastModTime, _ := item.LastMod()
if currentTime.(time.Time).Sub(itemLastModTime) < 1*time.Minute { if currentTime.(time.Time).Sub(itemLastModTime) < 1*time.Minute {
log.Debugf("item: %s created less than a minute ago: %v", item.ID(), itemLastModTime)
client.logger.Debug("item created less than a minute ago",
zap.String("item", item.ID()),
zap.Time("last_modified_time", itemLastModTime))
return true return true
} }
return false return false
storagesvc/util.go
+3 -3
View File
@@ -17,15 +17,15 @@ limitations under the License.
package storagesvc package storagesvc
import ( import (
"log"
"net/url" "net/url"
"github.com/pkg/errors"
) )
func getQueryParamValue(urlString string, queryParam string) (string, error) { func getQueryParamValue(urlString string, queryParam string) (string, error) {
url, err := url.Parse(urlString) url, err := url.Parse(urlString)
if err != nil { if err != nil {
log.Printf("Error parsing URL string: %s into URL", urlString) return "", errors.Wrapf(err, "error parsing URL string %q into URL", urlString)
return "", err
} }
return url.Query().Get(queryParam), nil return url.Query().Get(queryParam), nil
} }
test/test_utils.sh
+1 -1
View File
@@ -193,7 +193,7 @@ helm_install_fission() {
ns=f-$id ns=f-$id
fns=f-func-$id fns=f-func-$id
helmVars=repository=$repo,image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImage=$fluentdImage,logger.fluentdImageTag=$fluentdImageTag,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType,serviceType=$serviceType,preUpgradeChecksImage=$preUpgradeCheckImage,prometheus.server.persistentVolume.enabled=false,prometheus.alertmanager.enabled=false,prometheus.kubeStateMetrics.enabled=false,prometheus.nodeExporter.enabled=false helmVars=repository=$repo,image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImageRepository=$repo,logger.fluentdImage=$fluentdImage,logger.fluentdImageTag=$fluentdImageTag,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType,serviceType=$serviceType,preUpgradeChecksImage=$preUpgradeCheckImage,prometheus.server.persistentVolume.enabled=false,prometheus.alertmanager.enabled=false,prometheus.kubeStateMetrics.enabled=false,prometheus.nodeExporter.enabled=false
timeout 30 bash -c "helm_setup" timeout 30 bash -c "helm_setup"
test/upgrade/fission_upgrade_test.sh
+1 -1
View File
@@ -93,7 +93,7 @@ sudo mv $ROOT/fission/fission /usr/local/bin/
## Upgrade ## Upgrade
helmVars=repository=$repo,image=$IMAGE,imageTag=$TAG,fetcherImage=$FETCHER_IMAGE,fetcherImageTag=$TAG,logger.fluentdImage=$FLUENTD_IMAGE,logger.fluentdImageTag=$TAG,functionNamespace=$fns,controllerPort=$controllerNodeport,pullPolicy=Always,analytics=false,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType helmVars=repository=$repo,image=$IMAGE,imageTag=$TAG,fetcherImage=$FETCHER_IMAGE,fetcherImageTag=$TAG,logger.fluentdImageRepository=$repo,logger.fluentdImage=$FLUENTD_IMAGE,logger.fluentdImageTag=$TAG,functionNamespace=$fns,controllerPort=$controllerNodeport,pullPolicy=Always,analytics=false,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType
echo "Upgrading fission" echo "Upgrading fission"
helm upgrade \ helm upgrade \
timer/main.go
+7 -6
View File
@@ -17,25 +17,26 @@ limitations under the License.
package timer package timer
import ( import (
"log" "github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
"github.com/fission/fission/publisher" "github.com/fission/fission/publisher"
) )
func Start(routerUrl string) error { func Start(logger *zap.Logger, routerUrl string) error {
fissionClient, _, _, err := crd.MakeFissionClient() fissionClient, _, _, err := crd.MakeFissionClient()
if err != nil { if err != nil {
return err return errors.Wrap(err, "failed to get fission or kubernetes client")
} }
err = fissionClient.WaitForCRDs() err = fissionClient.WaitForCRDs()
if err != nil { if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err) return errors.Wrap(err, "error waiting for CRDs")
} }
poster := publisher.MakeWebhookPublisher(routerUrl) poster := publisher.MakeWebhookPublisher(logger, routerUrl)
MakeTimerSync(fissionClient, MakeTimer(poster)) MakeTimerSync(logger, fissionClient, MakeTimer(logger, poster))
return nil return nil
} }
timer/timer.go
+6 -5
View File
@@ -17,9 +17,8 @@ limitations under the License.
package timer package timer
import ( import (
"log"
"github.com/robfig/cron" "github.com/robfig/cron"
"go.uber.org/zap"
"github.com/fission/fission" "github.com/fission/fission"
"github.com/fission/fission/crd" "github.com/fission/fission/crd"
@@ -34,6 +33,7 @@ const (
type ( type (
Timer struct { Timer struct {
logger *zap.Logger
triggers map[string]*timerTriggerWithCron triggers map[string]*timerTriggerWithCron
requestChannel chan *timerRequest requestChannel chan *timerRequest
publisher *publisher.Publisher publisher *publisher.Publisher
@@ -53,8 +53,9 @@ type (
} }
) )
func MakeTimer(publisher publisher.Publisher) *Timer { func MakeTimer(logger *zap.Logger, publisher publisher.Publisher) *Timer {
timer := &Timer{ timer := &Timer{
logger: logger.Named("timer"),
triggers: make(map[string]*timerTriggerWithCron), triggers: make(map[string]*timerTriggerWithCron),
requestChannel: make(chan *timerRequest), requestChannel: make(chan *timerRequest),
publisher: &publisher, publisher: &publisher,
@@ -114,7 +115,7 @@ func (timer *Timer) syncCron(triggers []crd.TimeTrigger) error {
if _, found := triggerMap[k]; !found { if _, found := triggerMap[k]; !found {
if v.cron != nil { if v.cron != nil {
v.cron.Stop() v.cron.Stop()
log.Printf("Cron for time trigger %s stopped", v.trigger.Metadata.Name) timer.logger.Info("cron for time trigger stopped", zap.String("trigger", v.trigger.Metadata.Name))
} }
delete(timer.triggers, k) delete(timer.triggers, k)
} }
@@ -136,6 +137,6 @@ func (timer *Timer) newCron(t crd.TimeTrigger) *cron.Cron {
(*timer.publisher).Publish("", headers, fission.UrlForFunction(t.Spec.FunctionReference.Name, t.Metadata.Namespace)) (*timer.publisher).Publish("", headers, fission.UrlForFunction(t.Spec.FunctionReference.Name, t.Metadata.Namespace))
}) })
c.Start() c.Start()
log.Printf("Add new cron for time trigger %v", t.Metadata.Name) timer.logger.Info("added new cron for time trigger", zap.String("trigger", t.Metadata.Name))
return c return c
} }
timer/timerSync.go
+6 -4
View File
@@ -17,9 +17,9 @@ limitations under the License.
package timer package timer
import ( import (
"log"
"time" "time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission" "github.com/fission/fission"
@@ -28,13 +28,15 @@ import (
type ( type (
TimerSync struct { TimerSync struct {
logger *zap.Logger
fissionClient *crd.FissionClient fissionClient *crd.FissionClient
timer *Timer timer *Timer
} }
) )
func MakeTimerSync(fissionClient *crd.FissionClient, timer *Timer) *TimerSync { func MakeTimerSync(logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync {
ws := &TimerSync{ ws := &TimerSync{
logger: logger.Named("timer_sync"),
fissionClient: fissionClient, fissionClient: fissionClient,
timer: timer, timer: timer,
} }
@@ -47,11 +49,11 @@ func (ws *TimerSync) syncSvc() {
triggers, err := ws.fissionClient.TimeTriggers(metav1.NamespaceAll).List(metav1.ListOptions{}) triggers, err := ws.fissionClient.TimeTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil { if err != nil {
if fission.IsNetworkError(err) { if fission.IsNetworkError(err) {
log.Printf("Encounter network error, retry again: %v", err) ws.logger.Info("encountered a network error - will retry", zap.Error(err))
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
continue continue
} }
log.Fatalf("Failed get time trigger list: %v", err) ws.logger.Fatal("failed to get time trigger list", zap.Error(err))
} }
ws.timer.Sync(triggers.Items) ws.timer.Sync(triggers.Items)