Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use zap for logging (#1112)

Browse Source 
Use zap for logging
This commit is contained in:
Jon Carl
2019-03-14 21:11:15 +05:30
committed by Vishal
parent c717f182b6
commit 0fc864f230
98 changed files with 2031 additions and 1223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
builder/builder.go
+28 -25
View File
@@ -19,11 +19,9 @@ package builder
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"os/exec"
@@ -33,6 +31,8 @@ import (
"time"
"github.com/dchest/uniuri"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission"
)
@@ -60,12 +60,14 @@ type (
}
Builder struct {
logger *zap.Logger
sharedVolumePath string
}
)
func MakeBuilder(sharedVolumePath string) *Builder {
func MakeBuilder(logger *zap.Logger, sharedVolumePath string) *Builder {
return &Builder{
logger: logger.Named("builder"),
sharedVolumePath: sharedVolumePath,
}
}
@@ -77,37 +79,37 @@ func (builder *Builder) VersionHandler(w http.ResponseWriter, r *http.Request) {
func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
e := fmt.Sprintf("Method not allowed: %v", r.Method)
log.Println(e)
builder.reply(w, "", e, http.StatusMethodNotAllowed)
e := "method not allowed"
builder.logger.Error(e, zap.String("http_method", r.Method))
builder.reply(w, "", fmt.Sprintf("%s: %s", e, r.Method), http.StatusMethodNotAllowed)
return
}
startTime := time.Now()
defer func() {
elapsed := time.Since(startTime)
log.Printf("elapsed time in build request = %v", elapsed)
builder.logger.Info("build request complete", zap.Duration("elapsed_time", elapsed))
}()
// parse request
body, err := ioutil.ReadAll(r.Body)
if err != nil {
e := errors.New(fmt.Sprintf("Error reading request body: %v", err))
log.Println(e.Error())
builder.reply(w, "", e.Error(), http.StatusInternalServerError)
e := "error reading request body"
builder.logger.Error(e, zap.Error(err))
builder.reply(w, "", fmt.Sprintf("%s: %s", e, err.Error()), http.StatusInternalServerError)
return
}
var req PackageBuildRequest
err = json.Unmarshal(body, &req)
if err != nil {
e := errors.New(fmt.Sprintf("Error parsing json body: %v", err))
log.Println(e.Error())
builder.reply(w, "", e.Error(), http.StatusBadRequest)
e := "error parsing json body"
builder.logger.Error(e, zap.Error(err))
builder.reply(w, "", fmt.Sprintf("%s: %s", e, err.Error()), http.StatusBadRequest)
return
}
log.Printf("Builder received request: %v", req)
builder.logger.Info("builder received request", zap.Any("request", req))
log.Println("Starting build...")
builder.logger.Info("starting build")
srcPkgPath := filepath.Join(builder.sharedVolumePath, req.SrcPkgFilename)
deployPkgFilename := fmt.Sprintf("%v-%v", req.SrcPkgFilename, strings.ToLower(uniuri.NewLen(6)))
deployPkgPath := filepath.Join(builder.sharedVolumePath, deployPkgFilename)
@@ -118,10 +120,11 @@ func (builder *Builder) Handler(w http.ResponseWriter, r *http.Request) {
}
buildLogs, err := builder.build(buildCmd, srcPkgPath, deployPkgPath)
if err != nil {
e := errors.New(fmt.Sprintf("Error building source package: %v", err))
log.Println(e.Error())
e := "error building source package"
builder.logger.Error(e, zap.Error(err))
// append error at the end of build logs
buildLogs += fmt.Sprintf("%v\n", e.Error())
buildLogs += fmt.Sprintf("%s: %s\n", e, err.Error())
builder.reply(w, deployPkgFilename, buildLogs, http.StatusInternalServerError)
return
}
@@ -137,7 +140,7 @@ func (builder *Builder) reply(w http.ResponseWriter, pkgFilename string, buildLo
rBody, err := json.Marshal(resp)
if err != nil {
e := errors.New(fmt.Sprintf("Error encoding response body: %v", err))
e := errors.Wrap(err, "error encoding response body")
rBody = []byte(fmt.Sprintf(`{"buildLogs": "%v"}`, e.Error()))
statusCode = http.StatusInternalServerError
}
@@ -154,7 +157,7 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s
fi, err := os.Stat(srcPkgPath)
if err != nil {
return "", errors.New(fmt.Sprintf("could not find srcPkgPath: '%s'", srcPkgPath))
return "", fmt.Errorf("could not find srcPkgPath: '%s'", srcPkgPath)
}
if fi.IsDir() {
cmd.Dir = srcPkgPath
@@ -170,12 +173,12 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s
stdout, err := cmd.StdoutPipe()
if err != nil {
return "", errors.New(fmt.Sprintf("Error creating stdout pipe for cmd: %v", err.Error()))
return "", errors.Wrap(err, "error creating stdout pipe for cmd")
}
stderr, err := cmd.StderrPipe()
if err != nil {
return "", errors.New(fmt.Sprintf("Error creating stderr pipe for cmd: %v", err.Error()))
return "", errors.Wrap(err, "error creating stderr pipe for cmd")
}
var buildLogs string
@@ -190,7 +193,7 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s
err = cmd.Start()
if err != nil {
return "", errors.New(fmt.Sprintf("Error starting cmd: %v", err.Error()))
return "", errors.Wrap(err, "error starting cmd")
}
// Runtime logs
@@ -201,14 +204,14 @@ func (builder *Builder) build(command string, srcPkgPath string, deployPkgPath s
}
if err := scanner.Err(); err != nil {
scanErr := errors.New(fmt.Sprintf("Error reading cmd output: %v", err.Error()))
scanErr := errors.Wrap(err, "error reading cmd output")
fmt.Println(scanErr)
return buildLogs, scanErr
}
err = cmd.Wait()
if err != nil {
cmdErr := errors.New(fmt.Sprintf("Error waiting for cmd '%v': %v", command, err.Error()))
cmdErr := errors.Wrapf(err, "error waiting for cmd %q", command)
fmt.Println(cmdErr)
return buildLogs, cmdErr
}
builder/client/client.go
+12 -8
View File
@@ -20,31 +20,35 @@ import (
"bytes"
"encoding/json"
"io/ioutil"
"log"
"net/http"
"strings"
"time"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission"
builder "github.com/fission/fission/builder"
)
type (
Client struct {
url string
logger *zap.Logger
url string
}
)
func MakeClient(builderUrl string) *Client {
func MakeClient(logger *zap.Logger, builderUrl string) *Client {
return &Client{
url: strings.TrimSuffix(builderUrl, "/"),
logger: logger.Named("builder_client"),
url: strings.TrimSuffix(builderUrl, "/"),
}
}
func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildResponse, error) {
body, err := json.Marshal(req)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "error marshaling json")
}
maxRetries := 20
@@ -62,7 +66,7 @@ func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildR
if i < maxRetries-1 {
time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error building package (%v), retrying", err)
c.logger.Error("error building package, retrying", zap.Error(err))
continue
}
@@ -73,14 +77,14 @@ func (c *Client) Build(req *builder.PackageBuildRequest) (*builder.PackageBuildR
rBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Error reading resp body: %v", err)
c.logger.Error("error reading resp body", zap.Error(err))
return nil, err
}
pkgBuildResp := builder.PackageBuildResponse{}
err = json.Unmarshal([]byte(rBody), &pkgBuildResp)
if err != nil {
log.Printf("Error parsing resp body: %v", err)
c.logger.Error("error parsing resp body", zap.Error(err))
return nil, err
}
builder/cmd/main.go
+10 -2
View File
@@ -21,21 +21,29 @@ import (
"net/http"
"os"
"go.uber.org/zap"
builder "github.com/fission/fission/builder"
)
// Usage: builder <shared volume path>
func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
dir := os.Args[1]
if _, err := os.Stat(dir); err != nil {
if os.IsNotExist(err) {
err = os.MkdirAll(dir, os.ModeDir|0700)
if err != nil {
log.Fatalf("Error creating directory: %v", err)
logger.Fatal("error creating directory", zap.Error(err), zap.String("directory", dir))
}
}
}
builder := builder.MakeBuilder(dir)
builder := builder.MakeBuilder(logger, dir)
mux := http.NewServeMux()
mux.HandleFunc("/", builder.Handler)
mux.HandleFunc("/version", builder.VersionHandler)
buildermgr/buildermgr.go
+8 -7
View File
@@ -17,29 +17,30 @@ limitations under the License.
package buildermgr
import (
"log"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/crd"
)
// Start the buildermgr service.
func Start(storageSvcUrl string, envBuilderNamespace string) error {
func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
bmLogger := logger.Named("builder_manager")
fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
if err != nil {
log.Printf("Failed to get kubernetes client: %v", err)
return err
return errors.Wrap(err, "failed to get fission or kubernetes client")
}
err = fissionClient.WaitForCRDs()
if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err)
return errors.Wrap(err, "error waiting for CRDs")
}
envWatcher := makeEnvironmentWatcher(fissionClient, kubernetesClient, envBuilderNamespace)
envWatcher := makeEnvironmentWatcher(bmLogger, fissionClient, kubernetesClient, envBuilderNamespace)
go envWatcher.watchEnvironments()
pkgWatcher := makePackageWatcher(fissionClient,
pkgWatcher := makePackageWatcher(bmLogger, fissionClient,
kubernetesClient, envBuilderNamespace, storageSvcUrl)
go pkgWatcher.watchPackages(fissionClient, kubernetesClient, envBuilderNamespace)
buildermgr/common.go
+19 -16
View File
@@ -19,11 +19,13 @@ package buildermgr
import (
"context"
"fmt"
"log"
"net/http"
"strings"
"github.com/dchest/uniuri"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/builder"
builderClient "github.com/fission/fission/builder/client"
@@ -38,20 +40,21 @@ import (
// 3. Send upload request to fetcher to upload deployment package.
// 4. Return upload response and build logs.
// *. Return build logs and error if any one of steps above failed.
func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuilderNamespace string,
func buildPackage(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, envBuilderNamespace string,
storageSvcUrl string, pkg *crd.Package) (uploadResp *fission.ArchiveUploadResponse, buildLogs string, err error) {
env, err := fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
if err != nil {
e := fmt.Sprintf("Error getting environment CRD info: %v", err)
log.Println(e)
e := "error getting environment CRD info"
logger.Error(e, zap.Error(err))
e = fmt.Sprintf("%s: %v", e, err)
return nil, e, fission.MakeError(http.StatusInternalServerError, e)
}
svcName := fmt.Sprintf("%v-%v.%v", env.Metadata.Name, env.Metadata.ResourceVersion, envBuilderNamespace)
srcPkgFilename := fmt.Sprintf("%v-%v", pkg.Metadata.Name, strings.ToLower(uniuri.NewLen(6)))
fetcherC := fetcherClient.MakeClient(fmt.Sprintf("http://%v:8000", svcName))
builderC := builderClient.MakeClient(fmt.Sprintf("http://%v:8001", svcName))
fetcherC := fetcherClient.MakeClient(logger, fmt.Sprintf("http://%v:8000", svcName))
builderC := builderClient.MakeClient(logger, fmt.Sprintf("http://%v:8001", svcName))
fetchReq := &fission.FunctionFetchRequest{
FetchType: fission.FETCH_SOURCE,
@@ -63,8 +66,9 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
// send fetch request to fetcher
err = fetcherC.Fetch(ctx, fetchReq)
if err != nil {
e := fmt.Sprintf("Error fetching source package: %v", err)
log.Println(e)
e := "error fetching source package"
logger.Error(e, zap.Error(err))
e = fmt.Sprintf("%s: %v", e, err)
return nil, e, fission.MakeError(http.StatusInternalServerError, e)
}
@@ -78,12 +82,11 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
BuildCommand: buildCmd,
}
log.Printf("Start building with source package: %v", srcPkgFilename)
logger.Info("started building with source package", zap.String("source_package", srcPkgFilename))
// send build request to builder
buildResp, err := builderC.Build(pkgBuildReq)
if err != nil {
e := fmt.Sprintf("Error building deployment package: %v", err)
log.Println(e)
var buildLogs string
if buildResp != nil {
buildLogs = buildResp.BuildLogs
@@ -92,7 +95,7 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
return nil, buildLogs, fission.MakeError(http.StatusInternalServerError, e)
}
log.Printf("Build succeed, source package: %v, deployment package: %v", srcPkgFilename, buildResp.ArtifactFilename)
logger.Info("build succeed", zap.String("source_package", srcPkgFilename), zap.String("deployment_package", buildResp.ArtifactFilename))
archivePackage := !env.Spec.KeepArchive
@@ -102,12 +105,11 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
ArchivePackage: archivePackage,
}
log.Printf("Start uploading deployment package: %v", buildResp.ArtifactFilename)
logger.Info("started uploading deployment package", zap.String("deployment_package", buildResp.ArtifactFilename))
// ask fetcher to upload the deployment package
uploadResp, err = fetcherC.Upload(ctx, uploadReq)
if err != nil {
e := fmt.Sprintf("Error uploading deployment package: %v", err)
log.Println(e)
buildResp.BuildLogs += fmt.Sprintf("%v\n", e)
return nil, buildResp.BuildLogs, fission.MakeError(http.StatusInternalServerError, e)
}
@@ -115,7 +117,7 @@ func buildPackage(ctx context.Context, fissionClient *crd.FissionClient, envBuil
return uploadResp, buildResp.BuildLogs, nil
}
func updatePackage(fissionClient *crd.FissionClient,
func updatePackage(logger *zap.Logger, fissionClient *crd.FissionClient,
pkg *crd.Package, status fission.BuildStatus, buildLogs string,
uploadResp *fission.ArchiveUploadResponse) (*crd.Package, error) {
@@ -135,8 +137,9 @@ func updatePackage(fissionClient *crd.FissionClient,
// update package spec
pkg, err := fissionClient.Packages(pkg.Metadata.Namespace).Update(pkg)
if err != nil {
log.Printf("Error updating package: %v", err)
return nil, err
e := "error updating package"
logger.Error(e, zap.Error(err))
return nil, errors.Wrap(err, e)
}
// return resource version for function to update function package ref
buildermgr/envwatcher.go
+34 -28
View File
@@ -18,11 +18,12 @@ package buildermgr
import (
"fmt"
"log"
"os"
"strconv"
"time"
"github.com/pkg/errors"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -73,6 +74,7 @@ type (
}
environmentWatcher struct {
logger *zap.Logger
cache map[string]*builderInfo
requestChan chan envwRequest
builderNamespace string
@@ -86,7 +88,7 @@ type (
}
)
func makeEnvironmentWatcher(fissionClient *crd.FissionClient,
func makeEnvironmentWatcher(logger *zap.Logger, fissionClient *crd.FissionClient,
kubernetesClient *kubernetes.Clientset, builderNamespace string) *environmentWatcher {
useIstio := false
@@ -94,7 +96,7 @@ func makeEnvironmentWatcher(fissionClient *crd.FissionClient,
if len(enableIstio) > 0 {
istio, err := strconv.ParseBool(enableIstio)
if err != nil {
log.Println("Failed to parse ENABLE_ISTIO, defaults to false")
logger.Info("Failed to parse ENABLE_ISTIO, defaults to false")
}
useIstio = istio
}
@@ -109,6 +111,7 @@ func makeEnvironmentWatcher(fissionClient *crd.FissionClient,
collectorEndpoint := os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT")
envWatcher := &environmentWatcher{
logger: logger.Named("environment_watcher"),
cache: make(map[string]*builderInfo),
requestChan: make(chan envwRequest),
builderNamespace: builderNamespace,
@@ -153,11 +156,11 @@ func (envw *environmentWatcher) watchEnvironments() {
})
if err != nil {
if fission.IsNetworkError(err) {
log.Printf("Encounter network error, retrying later: %v", err)
envw.logger.Error("encountered network error, retrying later", zap.Error(err))
time.Sleep(5 * time.Second)
continue
}
log.Fatalf("Error watching environment list: %v", err)
envw.logger.Fatal("error watching environment list", zap.Error(err))
}
for {
@@ -185,11 +188,11 @@ func (envw *environmentWatcher) sync() {
envList, err := envw.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
if fission.IsNetworkError(err) {
log.Printf("Error syncing environment CRD resources due to network error, retrying later: %v", err)
envw.logger.Error("error syncing environment CRD resources due to network error, retrying later", zap.Error(err))
time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
continue
}
log.Fatalf("Error syncing environment CRD resources: %v", err)
envw.logger.Fatal("error syncing environment CRD resources", zap.Error(err))
}
// Create environment builders for all environments
@@ -202,7 +205,7 @@ func (envw *environmentWatcher) sync() {
}
_, err := envw.getEnvBuilder(&env)
if err != nil {
log.Printf("Error creating builder for %v: %v", env.Metadata.Name, err)
envw.logger.Error("error creating builder", zap.Error(err), zap.String("builder_target", env.Metadata.Name))
}
}
@@ -258,7 +261,7 @@ func (envw *environmentWatcher) service() {
svcList, err := envw.getBuilderServiceList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
if err != nil {
log.Println(err.Error())
envw.logger.Error("error getting the builder service list", zap.Error(err))
}
for _, svc := range svcList {
envName := svc.ObjectMeta.Labels[LABEL_ENV_NAME]
@@ -268,7 +271,9 @@ func (envw *environmentWatcher) service() {
if _, ok := latestEnvList[key]; !ok {
err := envw.deleteBuilderServiceByName(svc.ObjectMeta.Name, svc.ObjectMeta.Namespace)
if err != nil {
log.Printf("Error removing builder service: %v", err)
envw.logger.Error("error removing builder service", zap.Error(err),
zap.String("service_name", svc.ObjectMeta.Name),
zap.String("service_namespace", svc.ObjectMeta.Namespace))
}
}
delete(envw.cache, key)
@@ -276,7 +281,7 @@ func (envw *environmentWatcher) service() {
deployList, err := envw.getBuilderDeploymentList(envw.getLabelForDeploymentOwner(), metav1.NamespaceAll)
if err != nil {
log.Printf(err.Error())
envw.logger.Error("error getting the builder deployment list", zap.Error(err))
}
for _, deploy := range deployList {
envName := deploy.ObjectMeta.Labels[LABEL_ENV_NAME]
@@ -286,7 +291,9 @@ func (envw *environmentWatcher) service() {
if _, ok := latestEnvList[key]; !ok {
err := envw.deleteBuilderDeploymentByName(deploy.ObjectMeta.Name, deploy.ObjectMeta.Namespace)
if err != nil {
log.Printf("Error removing builder deployment: %v", err)
envw.logger.Error("error removing builder deployment", zap.Error(err),
zap.String("deployment_name", deploy.ObjectMeta.Name),
zap.String("deployment_namespace", deploy.ObjectMeta.Namespace))
}
}
delete(envw.cache, key)
@@ -327,12 +334,12 @@ func (envw *environmentWatcher) createBuilder(env *crd.Environment, ns string) (
if len(svcList) == 0 {
svc, err = envw.createBuilderService(env, ns)
if err != nil {
return nil, fmt.Errorf("Error creating builder service: %v", err)
return nil, errors.Wrap(err, "error creating builder service")
}
} else if len(svcList) == 1 {
svc = &svcList[0]
} else {
return nil, fmt.Errorf("Found more than one builder service for environment %v", env.Metadata.Name)
return nil, fmt.Errorf("found more than one builder service for environment %q", env.Metadata.Name)
}
deployList, err := envw.getBuilderDeploymentList(sel, ns)
@@ -344,18 +351,17 @@ func (envw *environmentWatcher) createBuilder(env *crd.Environment, ns string) (
// create builder SA in this ns, if not already created
_, err := fission.SetupSA(envw.kubernetesClient, fission.FissionBuilderSA, ns)
if err != nil {
log.Printf("Error : %v creating %s in ns : %s", err, fission.FissionBuilderSA, ns)
return nil, err
return nil, errors.Wrapf(err, "error creating %q in ns: %s", fission.FissionBuilderSA, ns)
}
deploy, err = envw.createBuilderDeployment(env, ns)
if err != nil {
return nil, fmt.Errorf("Error creating builder deployment: %v", err)
return nil, errors.Wrap(err, "error creating builder deployment")
}
} else if len(deployList) == 1 {
deploy = &deployList[0]
} else {
return nil, fmt.Errorf("Found more than one builder deployment for environment %v", env.Metadata.Name)
return nil, fmt.Errorf("found more than one builder deployment for environment %q", env.Metadata.Name)
}
return &builderInfo{
@@ -370,7 +376,7 @@ func (envw *environmentWatcher) deleteBuilderServiceByName(name, namespace strin
Services(namespace).
Delete(name, &delOpt)
if err != nil {
return fmt.Errorf("Error deleting builder service %s.%s: %v", name, namespace, err)
return errors.Wrapf(err, "error deleting builder service %s.%s", name, namespace)
}
return nil
}
@@ -380,7 +386,7 @@ func (envw *environmentWatcher) deleteBuilderDeploymentByName(name, namespace st
Deployments(namespace).
Delete(name, &delOpt)
if err != nil {
return fmt.Errorf("Error deleting builder deployment %s.%s: %v", name, namespace, err)
return errors.Wrapf(err, "error deleting builder deployment %s.%s", name, namespace)
}
return nil
}
@@ -391,12 +397,12 @@ func (envw *environmentWatcher) deleteBuilderService(sel map[string]string, ns s
return err
}
for _, svc := range svcList {
log.Printf("Removing builder service: %v", svc.ObjectMeta.Name)
envw.logger.Info("removing builder service", zap.String("service_name", svc.ObjectMeta.Name))
err = envw.kubernetesClient.CoreV1().
Services(ns).
Delete(svc.ObjectMeta.Name, &delOpt)
if err != nil {
return fmt.Errorf("Error deleting builder service: %v", err)
return errors.Wrap(err, "error deleting builder service")
}
}
return nil
@@ -408,12 +414,12 @@ func (envw *environmentWatcher) deleteBuilderDeployment(sel map[string]string, n
return err
}
for _, deploy := range deployList {
log.Printf("Removing builder deployment: %v", deploy.ObjectMeta.Name)
envw.logger.Info("removing builder deployment", zap.String("deployment_name", deploy.ObjectMeta.Name))
err = envw.kubernetesClient.ExtensionsV1beta1().
Deployments(ns).
Delete(deploy.ObjectMeta.Name, &delOpt)
if err != nil {
return fmt.Errorf("Error deleteing builder deployment: %v", err)
return errors.Wrap(err, "error deleteing builder deployment")
}
}
return nil
@@ -425,7 +431,7 @@ func (envw *environmentWatcher) getBuilderServiceList(sel map[string]string, ns
LabelSelector: labels.Set(sel).AsSelector().String(),
})
if err != nil {
return nil, fmt.Errorf("Error getting builder service list: %v", err)
return nil, errors.Wrap(err, "error getting builder service list")
}
return svcList.Items, nil
}
@@ -464,7 +470,7 @@ func (envw *environmentWatcher) createBuilderService(env *crd.Environment, ns st
},
},
}
log.Printf("Creating builder service: %v", name)
envw.logger.Info("creating builder service", zap.String("service_name", name))
_, err := envw.kubernetesClient.CoreV1().Services(ns).Create(&service)
if err != nil {
return nil, err
@@ -478,7 +484,7 @@ func (envw *environmentWatcher) getBuilderDeploymentList(sel map[string]string,
LabelSelector: labels.Set(sel).AsSelector().String(),
})
if err != nil {
return nil, fmt.Errorf("Error getting builder deployment list: %v", err)
return nil, errors.Wrap(err, "error getting builder deployment list")
}
return deployList.Items, nil
}
@@ -615,7 +621,7 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns
},
},
}
log.Printf("Creating builder deployment: %v", fmt.Sprintf("%v-%v", env.Metadata.Name, env.Metadata.ResourceVersion))
envw.logger.Info("creating builder deployment", zap.String("deployment", name))
_, err := envw.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Create(deployment)
if err != nil {
return nil, err
buildermgr/pkgwatcher.go
+45 -37
View File
@@ -19,23 +19,24 @@ package buildermgr
import (
"context"
"fmt"
"log"
"time"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/client-go/kubernetes"
k8sCache "k8s.io/client-go/tools/cache"
"github.com/fission/fission"
"github.com/fission/fission/cache"
"github.com/fission/fission/crd"
"k8s.io/client-go/kubernetes"
)
type (
packageWatcher struct {
logger *zap.Logger
fissionClient *crd.FissionClient
k8sClient *kubernetes.Clientset
podStore k8sCache.Store
@@ -45,13 +46,14 @@ type (
}
)
func makePackageWatcher(fissionClient *crd.FissionClient, k8sClientSet *kubernetes.Clientset,
func makePackageWatcher(logger *zap.Logger, fissionClient *crd.FissionClient, k8sClientSet *kubernetes.Clientset,
builderNamespace string, storageSvcUrl string) *packageWatcher {
lw := k8sCache.NewListWatchFromClient(k8sClientSet.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.Everything())
store, controller := k8sCache.NewInformer(lw, &apiv1.Pod{}, 30*time.Second, k8sCache.ResourceEventHandlerFuncs{})
go controller.Run(make(chan struct{}))
pkgw := &packageWatcher{
logger: logger.Named("package_watcher"),
fissionClient: fissionClient,
k8sClient: k8sClientSet,
podStore: store,
@@ -86,20 +88,20 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
}
defer buildCache.Delete(key)
log.Printf("Start build for package %v with resource version %v", srcpkg.Metadata.Name, srcpkg.Metadata.ResourceVersion)
pkgw.logger.Info("starting build for package", zap.String("package_name", srcpkg.Metadata.Name), zap.String("resource_version", srcpkg.Metadata.ResourceVersion))
pkg, err := updatePackage(pkgw.fissionClient, srcpkg, fission.BuildStatusRunning, "", nil)
pkg, err := updatePackage(pkgw.logger, pkgw.fissionClient, srcpkg, fission.BuildStatusRunning, "", nil)
if err != nil {
e := fmt.Sprintf("Error setting package pending state: %v", err)
log.Println(e)
updatePackage(pkgw.fissionClient, srcpkg, fission.BuildStatusFailed, e, nil)
pkgw.logger.Error("error setting package pending state", zap.Error(err))
return
}
env, err := pkgw.fissionClient.Environments(pkg.Spec.Environment.Namespace).Get(pkg.Spec.Environment.Name)
if errors.IsNotFound(err) {
updatePackage(pkgw.fissionClient, pkg,
fission.BuildStatusFailed, "Environment not existed", nil)
if k8serrors.IsNotFound(err) {
e := "environment does not exist"
pkgw.logger.Error(e, zap.String("environment", pkg.Spec.Environment.Name))
updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
fission.BuildStatusFailed, fmt.Sprintf("%s: %q", e, pkg.Spec.Environment.Name), nil)
return
}
@@ -109,12 +111,12 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
// iterate all available environment builders.
items := pkgw.podStore.List()
if err != nil {
log.Printf("Error retrieving pod information for env %v: %v", err, env.Metadata.Name)
pkgw.logger.Error("error retrieving pod information for environment", zap.Error(err), zap.String("environment", env.Metadata.Name))
return
}
if len(items) == 0 {
log.Printf("Environment \"%v\" builder pod is not existed yet, retry again later.", pkg.Spec.Environment.Name)
pkgw.logger.Info("builder pod does not exist for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name))
time.Sleep(time.Duration(i*1) * time.Second)
continue
}
@@ -145,7 +147,7 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
}
if !podIsReady {
log.Printf("Environment \"%v\" builder pod is not ready, retry again later.", pkg.Spec.Environment.Name)
pkgw.logger.Info("builder pod is not ready for environment, will retry again later", zap.String("environment", pkg.Spec.Environment.Name))
time.Sleep(time.Duration(i*1) * time.Second)
break
}
@@ -153,31 +155,37 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
// Add the package getter rolebinding to builder sa
// we continue here if role binding was not setup succeesffully. this is because without this, the fetcher wont be able to fetch the source pkg into the container and
// the build will fail eventually
err := fission.SetupRoleBinding(pkgw.k8sClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, builderNs)
err := fission.SetupRoleBinding(pkgw.logger, pkgw.k8sClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, builderNs)
if err != nil {
log.Printf("Error : %v in setting up the role binding %s for pkg : %s.%s", err, fission.PackageGetterRB, pkg.Metadata.Name, pkg.Metadata.Namespace)
pkgw.logger.Error("error setting up role binding for package",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namespace", pkg.Metadata.Namespace))
continue
} else {
log.Printf("Setup rolebinding for sa : %s.%s for pkg : %s.%s", fission.FissionBuilderSA, builderNs, pkg.Metadata.Name, pkg.Metadata.Namespace)
pkgw.logger.Info("setup rolebinding for sa package",
zap.String("sa", fmt.Sprintf("%s.%s", fission.FissionBuilderSA, builderNs)),
zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)))
}
ctx := context.Background()
uploadResp, buildLogs, err := buildPackage(ctx, pkgw.fissionClient, builderNs, pkgw.storageSvcUrl, pkg)
uploadResp, buildLogs, err := buildPackage(ctx, pkgw.logger, pkgw.fissionClient, builderNs, pkgw.storageSvcUrl, pkg)
if err != nil {
log.Printf("Error building package %v: %v", pkg.Metadata.Name, err)
updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
pkgw.logger.Error("error building package", zap.Error(err), zap.String("package_name", pkg.Metadata.Name))
updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
return
}
log.Printf("Start updating info of package: %v", pkg.Metadata.Name)
pkgw.logger.Info("starting package info update", zap.String("package_name", pkg.Metadata.Name))
fnList, err := pkgw.fissionClient.
Functions(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
e := fmt.Sprintf("Error getting function list: %v", err)
log.Println(e)
buildLogs += fmt.Sprintf("%v\n", e)
updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
e := "error getting function list"
pkgw.logger.Error(e, zap.Error(err))
buildLogs += fmt.Sprintf("%s: %v\n", e, err)
updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
}
// A package may be used by multiple functions. Update
@@ -190,33 +198,33 @@ func (pkgw *packageWatcher) build(buildCache *cache.Cache, srcpkg *crd.Package)
// update CRD
_, err = pkgw.fissionClient.Functions(fn.Metadata.Namespace).Update(&fn)
if err != nil {
e := fmt.Sprintf("Error updating function package resource version: %v", err)
log.Println(e)
buildLogs += fmt.Sprintf("%v\n", e)
updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
e := "error updating function package resource version"
pkgw.logger.Error(e, zap.Error(err))
buildLogs += fmt.Sprintf("%s: %v\n", e, err)
updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
return
}
}
}
_, err = updatePackage(pkgw.fissionClient, pkg,
_, err = updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
fission.BuildStatusSucceeded, buildLogs, uploadResp)
if err != nil {
log.Printf("Error update package info: %v", err)
updatePackage(pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
pkgw.logger.Error("error updating package info", zap.Error(err), zap.String("package_name", pkg.Metadata.Name))
updatePackage(pkgw.logger, pkgw.fissionClient, pkg, fission.BuildStatusFailed, buildLogs, nil)
return
}
log.Printf("Completed build request for package: %v", pkg.Metadata.Name)
pkgw.logger.Info("completed package build request", zap.String("package_name", pkg.Metadata.Name))
return
}
}
// build timeout
updatePackage(pkgw.fissionClient, pkg,
updatePackage(pkgw.logger, pkgw.fissionClient, pkg,
fission.BuildStatusFailed, "Build timeout due to environment builder not ready", nil)
log.Printf("Max retries exceeded in building the source pkg : %s.%s, timeout due to environment builder not ready",
pkg.Metadata.Name, pkg.Metadata.Namespace)
pkgw.logger.Error("max retries exceeded in building source package, timeout due to environment builder not ready",
zap.String("package", fmt.Sprintf("%s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)))
return
}
canaryconfigmgr/canaryConfigCache.go
+1 -1
View File
@@ -18,11 +18,11 @@ package canaryconfigmgr
import (
"context"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/cache"
"time"
)
type (
canaryconfigmgr/canaryConfigMgr.go
+163 -46
View File
@@ -24,7 +24,8 @@ import (
"strings"
"time"
log "github.com/sirupsen/logrus"
"github.com/pkg/errors"
"go.uber.org/zap"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
@@ -37,6 +38,7 @@ import (
)
type canaryConfigMgr struct {
logger *zap.Logger
fissionClient *crd.FissionClient
kubeClient *kubernetes.Clientset
canaryConfigStore k8sCache.Store
@@ -46,9 +48,9 @@ type canaryConfigMgr struct {
canaryCfgCancelFuncMap *canaryConfigCancelFuncMap
}
func MakeCanaryConfigMgr(fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, crdClient *rest.RESTClient, prometheusSvc string) (*canaryConfigMgr, error) {
func MakeCanaryConfigMgr(logger *zap.Logger, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, crdClient *rest.RESTClient, prometheusSvc string) (*canaryConfigMgr, error) {
if prometheusSvc == "" {
log.Info("Try to retrieve prometheus server information from environment variables")
logger.Info("try to retrieve prometheus server information from environment variables")
var prometheusSvcHost, prometheusSvcPort string
// handle a case where there is a prometheus server is already installed, try to find the service from env variable
@@ -73,12 +75,13 @@ func MakeCanaryConfigMgr(fissionClient *crd.FissionClient, kubeClient *kubernete
return nil, fmt.Errorf("prometheus service url not found/invalid, cant create canary config manager: %v", prometheusSvc)
}
promClient, err := MakePrometheusClient(prometheusSvc)
promClient, err := MakePrometheusClient(logger, prometheusSvc)
if err != nil {
return nil, err
}
configMgr := &canaryConfigMgr{
logger: logger.Named("canary_config_manager"),
fissionClient: fissionClient,
kubeClient: kubeClient,
crdClient: crdClient,
@@ -113,7 +116,10 @@ func (canaryCfgMgr *canaryConfigMgr) initCanaryConfigController() (k8sCache.Stor
newConfig := newObj.(*crd.CanaryConfig)
if oldConfig.Metadata.ResourceVersion != newConfig.Metadata.ResourceVersion &&
newConfig.Status.Status == fission.CanaryConfigStatusPending {
log.Printf("update canary config invoked for : %s.%s, newConfig.Status.Status=%s", newConfig.Metadata.Name, newConfig.Metadata.Namespace, newConfig.Status.Status)
canaryCfgMgr.logger.Info("update canary config invoked",
zap.String("name", newConfig.Metadata.Name),
zap.String("namespace", newConfig.Metadata.Namespace),
zap.String("version", newConfig.Metadata.ResourceVersion))
go canaryCfgMgr.updateCanaryConfig(oldConfig, newConfig)
}
go canaryCfgMgr.reSyncCanaryConfigs()
@@ -126,17 +132,21 @@ func (canaryCfgMgr *canaryConfigMgr) initCanaryConfigController() (k8sCache.Stor
func (canaryCfgMgr *canaryConfigMgr) Run(ctx context.Context) {
go canaryCfgMgr.canaryConfigController.Run(ctx.Done())
log.Printf("started Canary configmgr controller")
canaryCfgMgr.logger.Info("started canary configmgr controller")
}
func (canaryCfgMgr *canaryConfigMgr) addCanaryConfig(canaryConfig *crd.CanaryConfig) {
log.Printf("addCanaryConfig called for %s", canaryConfig.Metadata.Name)
canaryCfgMgr.logger.Info("addCanaryConfig called", zap.String("canary_config", canaryConfig.Metadata.Name))
// for each canary config, create a ticker with increment interval
interval, err := time.ParseDuration(canaryConfig.Spec.WeightIncrementDuration)
if err != nil {
log.Printf("Error parsing duration: %v, cant proceed with this canaryConfig : %v.%v", err,
canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace)
canaryCfgMgr.logger.Error("error parsing duration - cant proceed with this canaryConfig",
zap.Error(err),
zap.String("duration", canaryConfig.Spec.WeightIncrementDuration),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return
}
ticker := time.NewTicker(interval)
@@ -151,7 +161,11 @@ func (canaryCfgMgr *canaryConfigMgr) addCanaryConfig(canaryConfig *crd.CanaryCon
}
err = canaryCfgMgr.canaryCfgCancelFuncMap.assign(&canaryConfig.Metadata, cacheValue)
if err != nil {
log.Printf("Error caching canary config : %s.%s. err : %v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, err)
canaryCfgMgr.logger.Error("error caching canary config",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return
}
canaryCfgMgr.processCanaryConfig(&ctx, canaryConfig, ticker)
@@ -164,10 +178,17 @@ func (canaryCfgMgr *canaryConfigMgr) processCanaryConfig(ctx *context.Context, c
select {
case <-(*ctx).Done():
// this case when someone deleted their canary config in the middle of it being processed
log.Printf("Cancel Func called for canary config : %s", canaryConfig.Metadata.Name)
canaryCfgMgr.logger.Info("cancel func called for canary config",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata)
if err != nil {
log.Printf("error removing canary config: %s from map, err : %v", canaryConfig.Metadata.Name, err)
canaryCfgMgr.logger.Error("error removing canary config",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
}
return
@@ -175,16 +196,26 @@ func (canaryCfgMgr *canaryConfigMgr) processCanaryConfig(ctx *context.Context, c
// every weightIncrementDuration, check if failureThreshold has reached.
// if yes, rollback.
// else, increment the weight of new function and decrement old function by `weightIncrement`
log.Printf("Processing canary config : %s.%s", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace)
canaryCfgMgr.logger.Info("processing canary config",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
canaryCfgMgr.RollForwardOrBack(canaryConfig, quit, ticker)
case <-quit:
// we're done processing this canary config either because the new function receives 100% of the traffic
// or we rolled back to send all 100% traffic to old function
log.Printf("Quit processing canaryConfig : %s", canaryConfig.Metadata.Name)
canaryCfgMgr.logger.Info("quit processing canaryConfig",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&canaryConfig.Metadata)
if err != nil {
log.Printf("error removing canary config: %s from map, err : %v", canaryConfig.Metadata.Name, err)
canaryCfgMgr.logger.Error("error removing canary config from map",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
}
return
}
@@ -195,7 +226,10 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
// handle race between delete event and notification on ticker.C
_, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
if err != nil {
log.Printf("No need of processing the config, not in cache anymore")
canaryCfgMgr.logger.Info("no need of processing the config, not in cache anymore",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return
}
@@ -204,19 +238,31 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
if err != nil {
// if the http trigger is not found, then give up processing this config.
if k8serrors.IsNotFound(err) {
log.Printf("Http trigger object : %v.%v missing", canaryConfig.Spec.Trigger, canaryConfig.Metadata.Namespace)
canaryCfgMgr.logger.Error("http trigger object for canary config missing",
zap.Error(err),
zap.String("trigger", canaryConfig.Spec.Trigger),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
close(quit)
return
}
// just silently ignore. wait for next window to increment weight
log.Printf("Error fetching http trigger object, err : %v", err)
canaryCfgMgr.logger.Error("error fetching http trigger object for config",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return
}
// handle a race between ticker.Stop and receiving a notification on ticker.C
if canaryConfig.Status.Status != fission.CanaryConfigStatusPending {
log.Printf("No need of processing the config, not pending anymore")
canaryCfgMgr.logger.Info("no need of processing the config, not pending anymore",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return
}
@@ -227,22 +273,42 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
if err != nil {
// silently ignore. wait for next window to increment weight
log.Printf("Error calculating failure percentage, err : %v", err)
canaryCfgMgr.logger.Info("error calculating failure percentage",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return
}
log.Printf("Failure percentage calculated : %v for canaryConfig %s", failurePercent, canaryConfig.Metadata.Name)
canaryCfgMgr.logger.Info("failure percentage calculated for canaryConfig",
zap.Float64("failure_percent", failurePercent),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
if failurePercent == -1 {
// this means there were no requests triggered to this url during this window. return here and check back
// during next iteration
log.Printf("Total requests received for url : %v is 0", triggerObj.Spec.RelativeURL)
canaryCfgMgr.logger.Info("total requests received for url is 0", zap.String("url", triggerObj.Spec.RelativeURL))
return
}
if int(failurePercent) > canaryConfig.Spec.FailureThreshold {
log.Printf("Failure percent %v crossed the threshold %v, so rolling back", failurePercent, canaryConfig.Spec.FailureThreshold)
canaryCfgMgr.logger.Error("failure percent crossed the threshold, so rolling back",
zap.Float64("failure_percent", failurePercent),
zap.Int("threshold", canaryConfig.Spec.FailureThreshold),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
ticker.Stop()
canaryCfgMgr.rollback(canaryConfig, triggerObj)
err := canaryCfgMgr.rollback(canaryConfig, triggerObj)
if err != nil {
canaryCfgMgr.logger.Error("error rolling back canary config",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
}
close(quit)
return
}
@@ -251,7 +317,12 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
doneProcessingCanaryConfig, err := canaryCfgMgr.rollForward(canaryConfig, triggerObj)
if err != nil {
// just log the error and hope that next iteration will succeed
log.Printf("Error incrementing weights for triggerObj : %v, err : %v", triggerObj.Metadata.Name, err)
canaryCfgMgr.logger.Error("error incrementing weights for trigger",
zap.Error(err),
zap.String("trigger", triggerObj.Metadata.Name),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return
}
@@ -263,11 +334,17 @@ func (canaryCfgMgr *canaryConfigMgr) RollForwardOrBack(canaryConfig *crd.CanaryC
fission.CanaryConfigStatusSucceeded)
if err != nil {
// cant do much after max retries other than logging it.
log.Printf("Error updating canary config : %s.%s after max retries, err :%v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace,
err)
canaryCfgMgr.logger.Error("error updating canary config after max retries",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
}
log.Printf("We're done processing canary config : %s. The new function is receiving all the traffic", canaryConfig.Metadata.Name)
canaryCfgMgr.logger.Info("done processing canary config - the new function is receiving all the traffic",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
close(quit)
return
}
@@ -277,8 +354,9 @@ func (canaryCfgMgr *canaryConfigMgr) updateHttpTriggerWithRetries(triggerName, t
for i := 0; i < fission.MaxRetries; i++ {
triggerObj, err := canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Get(triggerName)
if err != nil {
log.Printf("Error getting http trigger object : %v", err)
return err
e := "error getting http trigger object"
canaryCfgMgr.logger.Error(e, zap.Error(err), zap.String("trigger_name", triggerName), zap.String("trigger_namespace", triggerNamespace))
return errors.Wrap(err, e)
}
triggerObj.Spec.FunctionReference.FunctionWeights = fnWeights
@@ -286,14 +364,21 @@ func (canaryCfgMgr *canaryConfigMgr) updateHttpTriggerWithRetries(triggerName, t
_, err = canaryCfgMgr.fissionClient.HTTPTriggers(triggerNamespace).Update(triggerObj)
switch {
case err == nil:
log.Printf("Updated Http trigger : %s.%s", triggerName, triggerNamespace)
canaryCfgMgr.logger.Info("updated http trigger", zap.String("trigger_name", triggerName), zap.String("trigger_namespace", triggerNamespace))
return nil
case k8serrors.IsConflict(err):
log.Printf("Conflict in updating http trigger : %s.%s, retrying", triggerName, triggerNamespace)
canaryCfgMgr.logger.Info("conflict in updating http trigger, retrying",
zap.Error(err),
zap.String("trigger_name", triggerName),
zap.String("trigger_namespace", triggerNamespace))
continue
default:
log.Printf("Error updating trigger : %s.%s = %v", triggerName, triggerNamespace, err)
return err
e := "error updating http trigger"
canaryCfgMgr.logger.Info(e,
zap.Error(err),
zap.String("trigger_name", triggerName),
zap.String("trigger_namespace", triggerNamespace))
return errors.Wrapf(err, "%s: %s.%s", e, triggerName, triggerNamespace)
}
}
@@ -304,24 +389,42 @@ func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfigStatusWithRetries(cfgName
for i := 0; i < fission.MaxRetries; i++ {
canaryCfgObj, err := canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Get(cfgName)
if err != nil {
log.Printf("Error getting http Canary Config object : %v", err)
return err
e := "error getting http canary config object"
canaryCfgMgr.logger.Error(e,
zap.Error(err),
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace),
zap.String("status", status))
return errors.Wrap(err, e)
}
log.Printf("Updating status of canaryCfg : %s.%s to %s", cfgName, cfgNamespace, status)
canaryCfgMgr.logger.Info("updating status of canary config",
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace),
zap.String("status", status))
canaryCfgObj.Status.Status = status
_, err = canaryCfgMgr.fissionClient.CanaryConfigs(cfgNamespace).Update(canaryCfgObj)
switch {
case err == nil:
log.Printf("Updated Canary Config : %s.%s", cfgName, cfgNamespace)
canaryCfgMgr.logger.Info("updated canary config",
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace))
return nil
case k8serrors.IsConflict(err):
log.Printf("Conflict in updating Canary Config : %s.%s, retrying", cfgName, cfgNamespace)
canaryCfgMgr.logger.Info("conflict in updating canary config",
zap.Error(err),
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace))
continue
default:
log.Printf("Error updating Canary Config : %s.%s = %v", cfgName, cfgNamespace, err)
return err
e := "error updating canary config"
canaryCfgMgr.logger.Error(e,
zap.Error(err),
zap.String("name", cfgName),
zap.String("namespace", cfgNamespace))
return errors.Wrapf(err, "%s: %s.%s", e, cfgName, cfgNamespace)
}
}
@@ -358,7 +461,7 @@ func (canaryCfgMgr *canaryConfigMgr) rollForward(canaryConfig *crd.CanaryConfig,
}
}
log.Printf("Incremented functionWeights : %v", functionWeights)
canaryCfgMgr.logger.Info("incremented functionWeights", zap.Any("function_weights", functionWeights))
err := canaryCfgMgr.updateHttpTriggerWithRetries(trigger.Metadata.Name, trigger.Metadata.Namespace, functionWeights)
return doneProcessingCanaryConfig, err
@@ -369,7 +472,10 @@ func (canaryCfgMgr *canaryConfigMgr) reSyncCanaryConfigs() {
canaryConfig := obj.(*crd.CanaryConfig)
_, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
if err != nil && canaryConfig.Status.Status == fission.CanaryConfigStatusPending {
log.Printf("Adding canary config : %s.%s from resync loop", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace)
canaryCfgMgr.logger.Info("adding canary config from resync loop",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
// new canaryConfig detected, add it to our cache and start processing it
go canaryCfgMgr.addCanaryConfig(canaryConfig)
@@ -378,10 +484,17 @@ func (canaryCfgMgr *canaryConfigMgr) reSyncCanaryConfigs() {
}
func (canaryCfgMgr *canaryConfigMgr) deleteCanaryConfig(canaryConfig *crd.CanaryConfig) {
log.Printf("Delete event received for canary config : %v, %v, %v", canaryConfig.Metadata.Name, canaryConfig.Metadata.Namespace, canaryConfig.Metadata.ResourceVersion)
canaryCfgMgr.logger.Info("delete event received for canary config",
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
canaryProcessingInfo, err := canaryCfgMgr.canaryCfgCancelFuncMap.lookup(&canaryConfig.Metadata)
if err != nil {
log.Printf("lookup of canaryConfig failed, err : %v", err)
canaryCfgMgr.logger.Error("lookup of canary config for deletion failed",
zap.Error(err),
zap.String("name", canaryConfig.Metadata.Name),
zap.String("namespace", canaryConfig.Metadata.Namespace),
zap.String("version", canaryConfig.Metadata.ResourceVersion))
return
}
// first stop the ticker
@@ -396,7 +509,11 @@ func (canaryCfgMgr *canaryConfigMgr) updateCanaryConfig(oldCanaryConfig *crd.Can
err := canaryCfgMgr.canaryCfgCancelFuncMap.remove(&oldCanaryConfig.Metadata)
if err != nil {
log.Printf("error removing canary config: %s from map, err : %v", oldCanaryConfig.Metadata.Name, err)
canaryCfgMgr.logger.Error("error removing canary config from map",
zap.Error(err),
zap.String("name", oldCanaryConfig.Metadata.Name),
zap.String("namespace", oldCanaryConfig.Metadata.Namespace),
zap.String("version", oldCanaryConfig.Metadata.ResourceVersion))
return
}
canaryCfgMgr.addCanaryConfig(newCanaryConfig)
canaryconfigmgr/prometheusClient.go
+27 -34
View File
@@ -20,26 +20,26 @@ import (
"fmt"
"time"
"golang.org/x/net/context"
"github.com/pkg/errors"
promClient "github.com/prometheus/client_golang/api/prometheus"
"github.com/prometheus/common/model"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
"golang.org/x/net/context"
)
type PrometheusApiClient struct {
logger *zap.Logger
client promClient.QueryAPI
}
func MakePrometheusClient(prometheusSvc string) (*PrometheusApiClient, error) {
func MakePrometheusClient(logger *zap.Logger, prometheusSvc string) (*PrometheusApiClient, error) {
promApiConfig := promClient.Config{
Address: prometheusSvc,
}
promApiClient, err := promClient.New(promApiConfig)
if err != nil {
log.Errorf("Error creating prometheus api client for svc : %s, err : %v", prometheusSvc, err)
return nil, err
return nil, errors.Wrapf(err, "error creating prometheus api client for svc: %s", prometheusSvc)
}
apiQueryClient := promClient.NewQueryAPI(promApiClient)
@@ -56,12 +56,12 @@ func MakePrometheusClient(prometheusSvc string) (*PrometheusApiClient, error) {
}
if err != nil {
log.Printf("Error sending test query to prometheus server: %v", err)
return nil, err
return nil, errors.Wrap(err, "error sending test query to prometheus server")
}
log.Printf("Successfully made prometheus client with service : %s", prometheusSvc)
logger.Info("successfully made prometheus client with service", zap.String("service", prometheusSvc))
return &PrometheusApiClient{
logger: logger.Named("prometheus_api_client"),
client: apiQueryClient,
}, nil
}
@@ -74,7 +74,7 @@ func (promApiClient *PrometheusApiClient) GetFunctionFailurePercentage(path, met
}
if reqs <= 0 {
return -1, fmt.Errorf("no requests to this url %v and method %v in the window : %v", path, method, window)
return -1, fmt.Errorf("no requests to this url %v and method %v in the window: %v", path, method, window)
}
// next, get a total count of errored out requests to this function in the same window
@@ -94,20 +94,22 @@ func (promApiClient *PrometheusApiClient) GetRequestsToFuncInWindow(path string,
reqs, err := promApiClient.executeQuery(queryString)
if err != nil {
log.Printf("Error executing query : %s, err : %v", queryString, err)
return 0, err
return 0, errors.Wrapf(err, "error executing query: %s", queryString)
}
queryString = fmt.Sprintf("fission_function_calls_total{path=\"%s\",method=\"%s\",name=\"%s\",namespace=\"%s\"} offset %v", path, method, funcName, funcNs, window)
reqsInPrevWindow, err := promApiClient.executeQuery(queryString)
if err != nil {
log.Printf("Error executing query : %s, err : %v", queryString, err)
return 0, err
return 0, errors.Wrapf(err, "error executing query: %s", queryString)
}
reqsInCurrentWindow := reqs - reqsInPrevWindow
log.Printf("reqs : %v, reqsInPrevWindow : %v, reqsInCurrentWindow : %v to function %v", reqs, reqsInPrevWindow, reqsInCurrentWindow, funcName)
promApiClient.logger.Info("function requests",
zap.Float64("requests", reqs),
zap.Float64("requests_in_previous_window", reqsInPrevWindow),
zap.Float64("requests_in_current_window", reqsInCurrentWindow),
zap.String("function", funcName))
return reqsInCurrentWindow, nil
}
@@ -117,20 +119,22 @@ func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(f
failedRequests, err := promApiClient.executeQuery(queryString)
if err != nil {
log.Printf("Error executing query : %s, err : %v", queryString, err)
return 0, err
return 0, errors.Wrapf(err, "error executing query: %s", queryString)
}
queryString = fmt.Sprintf("fission_function_errors_total{name=\"%s\",namespace=\"%s\",path=\"%s\", method=\"%s\"} offset %v", funcName, funcNs, path, method, window)
failedReqsInPrevWindow, err := promApiClient.executeQuery(queryString)
if err != nil {
log.Printf("Error executing query : %s, err : %v", queryString, err)
return 0, err
return 0, errors.Wrapf(err, "error executing query: %s", queryString)
}
failedReqsInCurrentWindow := failedRequests - failedReqsInPrevWindow
log.Printf("failedReqs : %v, failedReqsInPrevWindow : %v, failedReqsInCurrentWindow : %v to function : %v", failedRequests, failedReqsInPrevWindow, failedReqsInCurrentWindow, funcName)
promApiClient.logger.Info("function requests",
zap.Float64("failed_requests", failedRequests),
zap.Float64("failed_requests_in_previous_window", failedReqsInPrevWindow),
zap.Float64("failed_requests_in_current_window", failedReqsInCurrentWindow),
zap.String("function", funcName))
return failedReqsInCurrentWindow, nil
}
@@ -138,8 +142,7 @@ func (promApiClient *PrometheusApiClient) GetTotalFailedRequestsToFuncInWindow(f
func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (float64, error) {
val, err := promApiClient.client.Query(context.Background(), queryString, time.Now())
if err != nil {
log.Errorf("Error querying prometheus qs : %v, err : %v", queryString, err)
return 0, err
return 0, errors.Wrapf(err, "error querying prometheus")
}
switch {
@@ -159,23 +162,13 @@ func (promApiClient *PrometheusApiClient) executeQuery(queryString string) (floa
matrixVal := val.(model.Matrix)
total := float64(0)
for _, elem := range matrixVal {
//log.Printf("Only one value, so taking the 0th elem")
total += float64(elem.Values[len(elem.Values)-1].Value)
}
return total, nil
default:
log.Printf("type unrecognized")
promApiClient.logger.Info("return value type of prometheus query was unrecognized",
zap.Any("type", val.Type()))
return 0, nil
}
}
func addInterval(window string) string {
timeDuration, _ := time.ParseDuration(window)
log.Println("window in seconds", int64(timeDuration/time.Second))
timeInStr := fmt.Sprintf("%ds", int64((timeDuration+timeDuration)/time.Second))
fmt.Println(timeInStr)
return timeInStr
}
charts/fission-all/templates/fluentd.yaml
+1 -1
View File
@@ -28,7 +28,7 @@ spec:
spec:
containers:
- name: fluentd
image: "{{ .Values.repository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}"
image: "{{ .Values.logger.fluentdImageRepository }}/{{ .Values.logger.fluentdImage }}:{{ .Values.logger.fluentdImageTag }}"
imagePullPolicy: {{ .Values.pullPolicy }}
env:
- name: INFLUXDB_ADDRESS
charts/fission-all/values.yaml
+1
View File
@@ -53,6 +53,7 @@ enableIstio: false
## Logger config
logger:
influxdbAdmin: "admin"
fluentdImageRepository: index.docker.io
fluentdImage: fission/fluentd
fluentdImageTag: 1.0.0
common.go
+28 -8
View File
@@ -18,6 +18,7 @@ package fission
import (
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
@@ -28,6 +29,8 @@ import (
"strings"
"syscall"
"go.uber.org/zap"
"github.com/gorilla/handlers"
"github.com/imdario/mergo"
"github.com/mholt/archiver"
@@ -67,14 +70,31 @@ func GetFunctionIstioServiceName(fnName, fnNamespace string) string {
return fmt.Sprintf("istio-%v-%v", fnName, fnNamespace)
}
func LoggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestURI := r.RequestURI
if !strings.Contains(requestURI, "healthz") {
// Call the next handler, which can be another middleware in the chain, or the final handler.
handlers.LoggingHandler(os.Stdout, next).ServeHTTP(w, r)
}
})
func LoggingMiddleware(logger *zap.Logger) func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestURI := r.RequestURI
if !strings.Contains(requestURI, "healthz") {
// Call the next handler, which can be another middleware in the chain, or the final handler.
handlers.CustomLoggingHandler(os.Stdout, next, func(writer io.Writer, params handlers.LogFormatterParams) {
host, _, err := net.SplitHostPort(params.Request.RemoteAddr)
if err != nil {
host = params.Request.RemoteAddr
}
logger.Info("handled",
zap.String("host", host),
zap.String("method", params.Request.Method),
zap.String("uri", params.Request.RequestURI),
zap.String("proto", params.Request.Proto),
zap.Int("status_code", params.StatusCode),
zap.Int("size", params.Size))
}).ServeHTTP(w, r)
}
})
}
}
// MergeContainerSpecs merges container specs using a predefined order.
commonrbacutil.go
+62 -26
View File
@@ -18,8 +18,10 @@ package fission
import (
"fmt"
"log"
"go.uber.org/zap"
"github.com/pkg/errors"
apiv1 "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1beta1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -101,7 +103,7 @@ type PatchSpec struct {
}
// AddSaToRoleBindingWithRetries adds a service account to a rolebinding object. IT retries on already exists and conflict errors.
func AddSaToRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind string) (err error) {
func AddSaToRoleBindingWithRetries(logger *zap.Logger, k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind string) (err error) {
patch := PatchSpec{}
patch.Op = "add"
patch.Path = "/subjects/-"
@@ -113,34 +115,44 @@ func AddSaToRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding,
patchJson, err := json.Marshal([]PatchSpec{patch})
if err != nil {
log.Printf("Error marshalling patch into json")
logger.Error("error marshalling patch into json", zap.Error(err))
return err
}
for i := 0; i < MaxRetries; i++ {
_, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Patch(roleBinding, types.JSONPatchType, patchJson)
if err == nil {
log.Printf("Patched rolebinding : %s.%s", roleBinding, roleBindingNs)
logger.Info("patched rolebinding",
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return err
}
if k8serrors.IsNotFound(err) {
logger.Info("rolebinding not found - will try to create it",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
// someone may have deleted the object between us checking if the object is present and deciding to patch
// so just create the object again
rbObj := makeRoleBindingObj(roleBinding, roleBindingNs, role, roleKind, sa, saNamespace)
rbObj, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Create(rbObj)
if err == nil {
log.Printf("Created rolebinding : %s.%s", roleBinding, roleBindingNs)
logger.Info("created rolebinding",
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return err
}
if k8serrors.IsAlreadyExists(err) {
log.Printf("rolebinding object : %s.%s already exists, retrying patch, err: %v", roleBinding, roleBindingNs, err)
logger.Info("rolebinding object already exists, retrying patch",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
continue
}
log.Printf("Error returned by rolebinding create, %v", err)
return err
return errors.Wrap(err, "error returned by rolebinding create")
}
if k8serrors.IsConflict(err) {
@@ -149,27 +161,30 @@ func AddSaToRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding,
// but one CI run did show patch errored out on conflict : https://api.travis-ci.org/v3/job/373161490/log.txt, look for :
// Error returned by rolebinding patch : <some more text> there is a meaningful conflict (firstResourceVersion: "35482724", currentResourceVersion: "35482849")
// so, m guessing retrying patch should help. will watch out for any such conflicts and fix the issue if any
log.Printf("Conflict reported on patch of rolebinding : %s.%s. Retrying patch operation, err : %v", roleBinding, roleBindingNs, err)
logger.Info("conflict reported on patch of rolebinding - retrying patch operation",
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
continue
}
log.Printf("Error returned by rolebinding patch : %v", err)
return err
return errors.Wrap(err, "error returned by rolebinding patch")
}
log.Printf("Exceeded maxretries : %d adding SA: %s.%s to rolebinding : %s.%s, giving up, err: %v", MaxRetries, sa, saNamespace, roleBinding, roleBindingNs, err)
return err
return errors.Wrapf(err, "exceeded max retries (%d) adding SA: %s.%s to rolebinding: %s.%s, giving up", MaxRetries, sa, saNamespace, roleBinding, roleBindingNs)
}
// RemoveSAFromRoleBindingWithRetries removes an SA from the rolebinding passed as parameter. If this is the only SA in
// the rolebinding, then it deletes the rolebinding object.
func RemoveSAFromRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs string, saToRemove map[string]bool) (err error) {
func RemoveSAFromRoleBindingWithRetries(logger *zap.Logger, k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs string, saToRemove map[string]bool) (err error) {
for i := 0; i < MaxRetries; i++ {
rbObj, err := k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Get(
roleBinding, metav1.GetOptions{})
if err != nil {
// silently ignoring the error. there's no need for us to remove sa anymore.
log.Printf("rolebinding %s.%s not found, but ignoring the error since we're cleaning up", roleBinding, roleBindingNs)
logger.Info("rolebinding not found, but ignoring the error since we're cleaning up",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return nil
}
@@ -198,43 +213,64 @@ func RemoveSAFromRoleBindingWithRetries(k8sClient *kubernetes.Clientset, roleBin
_, err = k8sClient.RbacV1beta1().RoleBindings(rbObj.Namespace).Update(rbObj)
switch {
case err == nil:
log.Printf("Removed sa's : %v from rolebinding : %s.%s", saToRemove, roleBinding, roleBindingNs)
logger.Info("removed service accounts from rolebinding",
zap.Any("service_accounts", saToRemove),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return nil
case k8serrors.IsConflict(err):
log.Printf("Conflict in update of rolebinding: %s.%s, retrying", roleBinding, roleBindingNs)
logger.Info("conflict in update of rolebinding - retrying",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
continue
default:
log.Printf("Rolebinding Update Errored out : %v", err)
return err
return errors.Wrap(err, "rolebinding update errored out")
}
}
log.Printf("Maxretries : %d exceeded for removing SA's : %v from rolebinding %s.%s, giving up, err: %v", MaxRetries, saToRemove, roleBinding, roleBindingNs, err)
return err
return errors.Wrapf(err, "max retries: %d exceeded for removing SA's: %v from rolebinding %s.%s, giving up", MaxRetries, saToRemove, roleBinding, roleBindingNs)
}
// SetupRoleBinding adds a role to a service account if the rolebinding object is already present in the namespace.
// if not, it creates a rolebinding object granting the role to the SA in the namespace.
func SetupRoleBinding(k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, role, roleKind, sa, saNamespace string) error {
func SetupRoleBinding(logger *zap.Logger, k8sClient *kubernetes.Clientset, roleBinding, roleBindingNs, role, roleKind, sa, saNamespace string) error {
// get the role binding object
rbObj, err := k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Get(
roleBinding, metav1.GetOptions{})
if err == nil {
if !isSAInRoleBinding(rbObj, sa, saNamespace) {
return AddSaToRoleBindingWithRetries(k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind)
logger.Info("service account is not present in the rolebinding - will add",
zap.String("service_account_name", sa),
zap.String("service_account_namespace", saNamespace),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return AddSaToRoleBindingWithRetries(logger, k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind)
}
log.Printf("SA : %s.%s already present in rolebinding : %s.%s, so nothing to add", sa, saNamespace, roleBinding, roleBindingNs)
logger.Info("service account already present in rolebinding so nothing to add",
zap.String("service_account_name", sa),
zap.String("service_account_namespace", saNamespace),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
return nil
}
// if role binding is missing, create it. also add this sa to the binding.
if k8serrors.IsNotFound(err) {
log.Printf("Rolebinding %s does NOT exist in ns %s. Creating it", roleBinding, roleBindingNs)
logger.Info("rolebinding does NOT exist in namespace - creating it",
zap.Error(err),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
rbObj = makeRoleBindingObj(roleBinding, roleBindingNs, role, roleKind, sa, saNamespace)
rbObj, err = k8sClient.RbacV1beta1().RoleBindings(roleBindingNs).Create(rbObj)
if k8serrors.IsAlreadyExists(err) {
err = AddSaToRoleBindingWithRetries(k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind)
logger.Info("rolebinding already exists in namespace - adding service account to rolebinding",
zap.String("service_account_name", sa),
zap.String("service_account_namespace", saNamespace),
zap.String("role_binding", roleBinding),
zap.String("role_binding_namespace", roleBindingNs))
err = AddSaToRoleBindingWithRetries(logger, k8sClient, roleBinding, roleBindingNs, sa, saNamespace, role, roleKind)
}
}
controller/api.go
+9 -7
View File
@@ -24,7 +24,7 @@ import (
"strings"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,6 +46,7 @@ func init() {
type (
API struct {
logger *zap.Logger
fissionClient *crd.FissionClient
kubernetesClient *kubernetes.Clientset
storageServiceUrl string
@@ -63,8 +64,8 @@ type (
}
)
func MakeAPI(featureStatus map[string]string) (*API, error) {
api, err := makeCRDBackedAPI()
func MakeAPI(logger *zap.Logger, featureStatus map[string]string) (*API, error) {
api, err := makeCRDBackedAPI(logger)
u := os.Getenv("STORAGE_SERVICE_URL")
if len(u) > 0 {
@@ -119,7 +120,7 @@ func (api *API) respondWithError(w http.ResponseWriter, err error) {
}
code, msg := fission.GetHTTPError(err)
log.Errorf("Error: %v: %v", code, msg)
api.logger.Error(msg, zap.Int("code", code))
http.Error(w, msg, code)
}
@@ -270,7 +271,8 @@ func (api *API) Serve(port int) {
address := fmt.Sprintf(":%v", port)
log.WithFields(log.Fields{"port": port}).Info("Server started")
r.Use(fission.LoggingMiddleware)
log.Fatal(http.ListenAndServe(address, r))
api.logger.Info("server started", zap.Int("port", port))
r.Use(fission.LoggingMiddleware(api.logger))
err := http.ListenAndServe(address, r)
api.logger.Fatal("done listening", zap.Error(err))
}
controller/api_test.go
+5 -1
View File
@@ -28,6 +28,7 @@ import (
"testing"
"time"
"go.uber.org/zap"
"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -344,7 +345,10 @@ func TestMain(m *testing.M) {
return
}
go Start(8888, true)
logger, err := zap.NewDevelopment()
panicIf(err)
go Start(logger, 8888, true)
time.Sleep(5 * time.Second)
g.client = client.MakeClient("http://localhost:8888")
controller/canaryConfigApi.go
+2 -2
View File
@@ -23,7 +23,7 @@ import (
"net/http"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
@@ -47,7 +47,7 @@ func (a *API) CanaryConfigApiCreate(w http.ResponseWriter, r *http.Request) {
var canaryCfg crd.CanaryConfig
err = json.Unmarshal(body, &canaryCfg)
if err != nil {
log.Printf("Failed to unmarshal request body: [%v]", body)
a.logger.Error("failed to unmarshal request body", zap.Error(err), zap.Binary("body", body))
a.respondWithError(w, err)
return
}
controller/config.go
+9 -9
View File
@@ -18,9 +18,9 @@ package controller
import (
"context"
"fmt"
log "github.com/sirupsen/logrus"
"github.com/pkg/errors"
"go.uber.org/zap"
"k8s.io/client-go/kubernetes"
"github.com/fission/fission/canaryconfigmgr"
@@ -28,24 +28,24 @@ import (
config "github.com/fission/fission/featureconfig"
)
func ConfigCanaryFeature(context context.Context, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, featureConfig *config.FeatureConfig, featureStatus map[string]string) error {
func ConfigCanaryFeature(context context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset, featureConfig *config.FeatureConfig, featureStatus map[string]string) error {
// start the appropriate controller
if featureConfig.CanaryConfig.IsEnabled {
canaryCfgMgr, err := canaryconfigmgr.MakeCanaryConfigMgr(fissionClient, kubeClient, fissionClient.GetCrdClient(),
canaryCfgMgr, err := canaryconfigmgr.MakeCanaryConfigMgr(logger, fissionClient, kubeClient, fissionClient.GetCrdClient(),
featureConfig.CanaryConfig.PrometheusSvc)
if err != nil {
featureStatus[config.CanaryFeature] = err.Error()
return fmt.Errorf("failed to start canary config manager: %v", err)
return errors.Wrap(err, "failed to start canary config manager")
}
canaryCfgMgr.Run(context)
log.Printf("Started canary config manager")
logger.Info("started canary config manager")
}
return nil
}
// ConfigureFeatures gets the feature config and configures the features that are enabled
func ConfigureFeatures(context context.Context, unitTestMode bool, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset) (map[string]string, error) {
func ConfigureFeatures(context context.Context, logger *zap.Logger, unitTestMode bool, fissionClient *crd.FissionClient, kubeClient *kubernetes.Clientset) (map[string]string, error) {
// set feature enabled to false if unitTestMode
if unitTestMode {
return nil, nil
@@ -54,7 +54,7 @@ func ConfigureFeatures(context context.Context, unitTestMode bool, fissionClient
// get the featureConfig from config map mounted onto the file system
featureConfig, err := config.GetFeatureConfig()
if err != nil {
log.Printf("Error getting feature config : %v", err)
logger.Error("error getting feature config", zap.Error(err))
return nil, err
}
@@ -62,6 +62,6 @@ func ConfigureFeatures(context context.Context, unitTestMode bool, fissionClient
// configure respective features
// in the future when new optional features are added, we need to add corresponding feature handlers and invoke them here
err = ConfigCanaryFeature(context, fissionClient, kubeClient, featureConfig, featureStatus)
err = ConfigCanaryFeature(context, logger, fissionClient, kubeClient, featureConfig, featureStatus)
return featureStatus, err
}
controller/configmapApi.go
+2 -2
View File
@@ -21,7 +21,7 @@ import (
"net/http"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -35,7 +35,7 @@ func (a *API) ConfigMapGet(w http.ResponseWriter, r *http.Request) {
configMap, err := a.kubernetesClient.CoreV1().ConfigMaps(ns).Get(name, metav1.GetOptions{})
if err != nil {
log.Printf("Error getting config map: %s from ns: %s", name, ns)
a.logger.Error("error getting config map", zap.Error(err), zap.String("config_map_name", name), zap.String("namespace", ns))
a.respondWithError(w, err)
return
}
controller/controller.go
+12 -10
View File
@@ -18,41 +18,43 @@ package controller
import (
"context"
"log"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/crd"
)
func Start(port int, unitTestFlag bool) {
func Start(logger *zap.Logger, port int, unitTestFlag bool) {
cLogger := logger.Named("controller")
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
fc, kc, apiExtClient, err := crd.MakeFissionClient()
if err != nil {
log.Fatalf("Failed to connect to K8s API: %v", err)
cLogger.Fatal("failed to connect to k8s API", zap.Error(err))
}
err = crd.EnsureFissionCRDs(apiExtClient)
err = crd.EnsureFissionCRDs(cLogger, apiExtClient)
if err != nil {
log.Fatalf("Failed to create fission CRDs: %v", err)
cLogger.Fatal("failed to create fission CRDs", zap.Error(err))
}
err = fc.WaitForCRDs()
if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err)
cLogger.Fatal("error waiting for CRDs", zap.Error(err))
}
ctx, cancel := context.WithCancel(context.Background())
featureStatus, err := ConfigureFeatures(ctx, unitTestFlag, fc, kc)
featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc)
if err != nil {
log.Printf("Error configuring features : %v. Proceeding without optional features", err.Error())
cLogger.Info("error configuring features - proceeding without optional features", zap.Error(err))
}
defer cancel()
api, err := MakeAPI(featureStatus)
api, err := MakeAPI(cLogger, featureStatus)
if err != nil {
log.Fatalf("Failed to start controller: %v", err)
cLogger.Fatal("failed to start controller", zap.Error(err))
}
api.Serve(port)
}
controller/crd.go
+8 -2
View File
@@ -17,13 +17,19 @@ limitations under the License.
package controller
import (
"go.uber.org/zap"
"github.com/fission/fission/crd"
)
func makeCRDBackedAPI() (*API, error) {
func makeCRDBackedAPI(logger *zap.Logger) (*API, error) {
fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
if err != nil {
return nil, err
}
return &API{fissionClient: fissionClient, kubernetesClient: kubernetesClient}, nil
return &API{
logger: logger.Named("api"),
fissionClient: fissionClient,
kubernetesClient: kubernetesClient,
}, nil
}
controller/environmentApi.go
+2 -2
View File
@@ -22,7 +22,7 @@ import (
"net/http"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
@@ -60,7 +60,7 @@ func (a *API) EnvironmentApiCreate(w http.ResponseWriter, r *http.Request) {
var env crd.Environment
err = json.Unmarshal(body, &env)
if err != nil {
log.Printf("Failed to unmarshal request body: [%v]", body)
a.logger.Error("failed to unmarshal request body", zap.Error(err), zap.Binary("body", body))
a.respondWithError(w, err)
return
}
controller/functionApi.go
+4 -2
View File
@@ -27,7 +27,7 @@ import (
"sort"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
restclient "k8s.io/client-go/rest"
@@ -188,7 +188,9 @@ func (a *API) FunctionLogsApiPost(w http.ResponseWriter, r *http.Request) {
svcUrl, err := url.Parse(dbCnf.httpURL)
if err != nil {
log.Printf("Failed to establish proxy server for function logs: %v", err)
a.logger.Error("failed parse url to establish proxy to database for function logs",
zap.Error(err),
zap.String("database_url", dbCnf.httpURL))
}
// set up proxy server director
director := func(req *http.Request) {
controller/recordsApi.go
+4 -4
View File
@@ -26,7 +26,7 @@ import (
)
func (a *API) RecordsApiListAll(w http.ResponseWriter, r *http.Request) {
resp, err := redis.RecordsListAll()
resp, err := redis.RecordsListAll(a.logger.Named("redis"))
if err != nil {
a.respondWithError(w, err)
return
@@ -50,7 +50,7 @@ func (a *API) RecordsApiFilterByFunction(w http.ResponseWriter, r *http.Request)
return
}
resp, err := redis.RecordsFilterByFunction(query, recorders, triggers)
resp, err := redis.RecordsFilterByFunction(a.logger.Named("redis"), query, recorders, triggers)
if err != nil {
a.respondWithError(w, err)
return
@@ -74,7 +74,7 @@ func (a *API) RecordsApiFilterByTrigger(w http.ResponseWriter, r *http.Request)
return
}
resp, err := redis.RecordsFilterByTrigger(query, recorders, triggers)
resp, err := redis.RecordsFilterByTrigger(a.logger.Named("redis"), query, recorders, triggers)
if err != nil {
a.respondWithError(w, err)
return
@@ -86,7 +86,7 @@ func (a *API) RecordsApiFilterByTime(w http.ResponseWriter, r *http.Request) {
from := r.FormValue("from")
to := r.FormValue("to")
resp, err := redis.RecordsFilterByTime(from, to)
resp, err := redis.RecordsFilterByTime(a.logger.Named("redis"), from, to)
if err != nil {
a.respondWithError(w, err)
return
controller/replayAPI.go
+1 -1
View File
@@ -31,7 +31,7 @@ func (a *API) ReplayByReqUID(w http.ResponseWriter, r *http.Request) {
routerUrl := fmt.Sprintf("http://router.%v", podNamespace)
resp, err := redis.ReplayByReqUID(routerUrl, queriedID)
resp, err := redis.ReplayByReqUID(a.logger, routerUrl, queriedID)
if err != nil {
a.respondWithError(w, err)
return
controller/secretApi.go
+5 -2
View File
@@ -21,7 +21,7 @@ import (
"net/http"
"github.com/gorilla/mux"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -35,7 +35,10 @@ func (a *API) SecretGet(w http.ResponseWriter, r *http.Request) {
secret, err := a.kubernetesClient.CoreV1().Secrets(ns).Get(name, metav1.GetOptions{})
if err != nil {
log.Printf("Error getting secret: %s from ns: %s", name, ns)
a.logger.Error("error getting secret",
zap.Error(err),
zap.String("secret_name", name),
zap.String("namespace", ns))
a.respondWithError(w, err)
return
}
controller/storagesvc.go
+5 -4
View File
@@ -18,19 +18,20 @@ package controller
import (
"fmt"
"log"
"net/http"
"net/http/httputil"
"net/url"
"go.uber.org/zap"
)
func (api *API) StorageServiceProxy(w http.ResponseWriter, r *http.Request) {
u := api.storageServiceUrl
ssUrl, err := url.Parse(u)
if err != nil {
msg := fmt.Sprintf("Error parsing url %v: %v", u, err)
log.Println(msg)
http.Error(w, msg, http.StatusInternalServerError)
e := "error parsing url"
api.logger.Error(e, zap.Error(err), zap.String("url", u))
http.Error(w, fmt.Sprintf("%s %s: %v", e, u, err), http.StatusInternalServerError)
return
}
director := func(req *http.Request) {
controller/workflowApiProxy.go
+2 -1
View File
@@ -2,10 +2,11 @@ package controller
import (
"fmt"
"github.com/gorilla/mux"
"net/http"
"net/http/httputil"
"net/url"
"github.com/gorilla/mux"
)
func (api *API) WorkflowApiserverProxy(w http.ResponseWriter, r *http.Request) {
crd/crd.go
+7 -7
View File
@@ -17,12 +17,12 @@ limitations under the License.
package crd
import (
"log"
"time"
"go.uber.org/zap"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/api/errors"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -34,7 +34,7 @@ const (
// ensureCRD checks if the given CRD type exists, and creates it if
// needed. (Note that this creates the CRD type; it doesn't create any
// _instances_ of that type.)
func ensureCRD(clientset *apiextensionsclient.Clientset, crd *apiextensionsv1beta1.CustomResourceDefinition) (err error) {
func ensureCRD(logger *zap.Logger, clientset *apiextensionsclient.Clientset, crd *apiextensionsv1beta1.CustomResourceDefinition) (err error) {
maxRetries := 5
for i := 0; i < maxRetries; i++ {
@@ -44,12 +44,12 @@ func ensureCRD(clientset *apiextensionsclient.Clientset, crd *apiextensionsv1bet
}
// return if the resource already exists
if errors.IsAlreadyExists(err) {
if k8serrors.IsAlreadyExists(err) {
return nil
} else {
// The requests fail to connect to k8s api server before
// istio-prxoy is ready to serve traffic. Retry again.
log.Printf("Error connecting to kubernetes api service (%v), retrying", err)
logger.Info("error connecting to kubernetes api service, retrying", zap.Error(err))
time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
continue
}
@@ -59,7 +59,7 @@ func ensureCRD(clientset *apiextensionsclient.Clientset, crd *apiextensionsv1bet
}
// Ensure CRDs
func EnsureFissionCRDs(clientset *apiextensionsclient.Clientset) error {
func EnsureFissionCRDs(logger *zap.Logger, clientset *apiextensionsclient.Clientset) error {
crds := []apiextensionsv1beta1.CustomResourceDefinition{
// Functions
{
@@ -207,7 +207,7 @@ func EnsureFissionCRDs(clientset *apiextensionsclient.Clientset) error {
},
}
for _, crd := range crds {
err := ensureCRD(clientset, &crd)
err := ensureCRD(logger, clientset, &crd)
if err != nil {
return err
}
crd/crd_test.go
+5 -1
View File
@@ -22,6 +22,7 @@ import (
"testing"
"time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/rest"
@@ -427,8 +428,11 @@ func TestCrd(t *testing.T) {
config, _, apiExtClient, err := GetKubernetesClient()
panicIf(err)
logger, err := zap.NewDevelopment()
panicIf(err)
// init our types
err = EnsureFissionCRDs(apiExtClient)
err = EnsureFissionCRDs(logger, apiExtClient)
panicIf(err)
// rest client with knowledge about our crd types
crd/recorder.go
-1
View File
@@ -19,7 +19,6 @@ package crd
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
)
environments/fetcher/client/client.go
+13 -9
View File
@@ -5,7 +5,9 @@ import (
"context"
"encoding/json"
"io/ioutil"
"log"
"go.uber.org/zap"
"net/http"
"strings"
"time"
@@ -18,14 +20,16 @@ import (
type (
Client struct {
logger *zap.Logger
url string
httpClient *http.Client
}
)
func MakeClient(fetcherUrl string) *Client {
func MakeClient(logger *zap.Logger, fetcherUrl string) *Client {
return &Client{
url: strings.TrimSuffix(fetcherUrl, "/"),
logger: logger.Named("fetcher_client"),
url: strings.TrimSuffix(fetcherUrl, "/"),
httpClient: &http.Client{
Transport: &ochttp.Transport{},
},
@@ -45,17 +49,17 @@ func (c *Client) getUploadUrl() string {
}
func (c *Client) Specialize(ctx context.Context, req *fission.FunctionSpecializeRequest) error {
_, err := sendRequest(ctx, c.httpClient, req, c.getSpecializeUrl())
_, err := sendRequest(c.logger, ctx, c.httpClient, req, c.getSpecializeUrl())
return err
}
func (c *Client) Fetch(ctx context.Context, fr *fission.FunctionFetchRequest) error {
_, err := sendRequest(ctx, c.httpClient, fr, c.getFetchUrl())
_, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getFetchUrl())
return err
}
func (c *Client) Upload(ctx context.Context, fr *fission.ArchiveUploadRequest) (*fission.ArchiveUploadResponse, error) {
body, err := sendRequest(ctx, c.httpClient, fr, c.getUploadUrl())
body, err := sendRequest(c.logger, ctx, c.httpClient, fr, c.getUploadUrl())
uploadResp := fission.ArchiveUploadResponse{}
err = json.Unmarshal(body, &uploadResp)
@@ -66,7 +70,7 @@ func (c *Client) Upload(ctx context.Context, fr *fission.ArchiveUploadRequest) (
return &uploadResp, nil
}
func sendRequest(ctx context.Context, httpClient *http.Client, req interface{}, url string) ([]byte, error) {
func sendRequest(logger *zap.Logger, ctx context.Context, httpClient *http.Client, req interface{}, url string) ([]byte, error) {
body, err := json.Marshal(req)
if err != nil {
return nil, err
@@ -82,7 +86,7 @@ func sendRequest(ctx context.Context, httpClient *http.Client, req interface{},
if resp.StatusCode == 200 {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Error reading response body: %v", err)
logger.Error("error reading response body", zap.Error(err))
}
resp.Body.Close()
return body, err
@@ -92,7 +96,7 @@ func sendRequest(ctx context.Context, httpClient *http.Client, req interface{},
if i < maxRetries-1 {
time.Sleep(50 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error specializing/fetching/uploading package (%v) with url %v, retrying", err, url)
logger.Error("error specializing/fetching/uploading package, retrying", zap.Error(err), zap.String("url", url))
continue
}
}
environments/fetcher/cmd/main.go
+17 -11
View File
@@ -12,6 +12,8 @@ import (
"runtime/debug"
"syscall"
"go.uber.org/zap"
"go.opencensus.io/exporter/jaeger"
"go.opencensus.io/plugin/ochttp"
"go.opencensus.io/trace"
@@ -49,14 +51,18 @@ func registerTraceExporter(collectorEndpoint string) error {
// Usage: fetcher <shared volume path>
func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
// register signal handler for dumping stack trace.
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGTERM)
go func() {
<-c
log.Println("Received SIGTERM : Dumping stack trace")
dumpStackTrace()
os.Exit(1)
logger.Fatal("received SIGTERM")
}()
flag.Usage = fetcherUsage
@@ -77,18 +83,18 @@ func main() {
if os.IsNotExist(err) {
err = os.MkdirAll(dir, os.ModeDir|0700)
if err != nil {
log.Fatalf("Error creating directory: %v", err)
logger.Fatal("error creating directory", zap.Error(err), zap.String("directory", dir))
}
}
}
if err := registerTraceExporter(*collectorEndpoint); err != nil {
log.Fatalf("Could not register trace exporter: %v", err)
logger.Fatal("could not register trace exporter", zap.Error(err), zap.String("collector_endpoint", *collectorEndpoint))
}
f, err := fetcher.MakeFetcher(dir, *secretDir, *configDir)
f, err := fetcher.MakeFetcher(logger, dir, *secretDir, *configDir)
if err != nil {
log.Fatalf("Error making fetcher: %v", err)
logger.Fatal("error making fetcher", zap.Error(err))
}
readyToServe := false
@@ -100,13 +106,13 @@ func main() {
err := json.Unmarshal([]byte(*specializePayload), &specializeReq)
if err != nil {
log.Fatalf("Error decoding specialize request: %v", err)
logger.Fatal("error decoding specialize request", zap.Error(err))
}
ctx := context.Background()
err = f.SpecializePod(ctx, specializeReq.FetchReq, specializeReq.LoadReq)
if err != nil {
log.Fatalf("Error specialing function poadt: %v", err)
logger.Fatal("error specializing function pod", zap.Error(err))
}
readyToServe = true
@@ -129,12 +135,12 @@ func main() {
w.WriteHeader(http.StatusOK)
})
log.Println("Fetcher ready to receive requests")
logger.Info("fetcher ready to receive requests")
http.ListenAndServe(":8000", &ochttp.Handler{
Handler: mux,
})
}
func fetcherUsage() {
fmt.Printf("Usage: fetcher [-specialize-on-startup] [-specialize-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path> \n")
fmt.Println("Usage: fetcher [-specialize-on-startup] [-specialize-request <json>] [-secret-dir <string>] [-cfgmap-dir <string>] <shared volume path>")
}
environments/fetcher/fetcher.go
+144 -93
View File
@@ -9,7 +9,6 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"net/url"
@@ -17,6 +16,8 @@ import (
"path/filepath"
"time"
"go.uber.org/zap"
"github.com/mholt/archiver"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
@@ -33,6 +34,7 @@ import (
type (
Fetcher struct {
logger *zap.Logger
sharedVolumePath string
sharedSecretPath string
sharedConfigPath string
@@ -42,23 +44,31 @@ type (
}
)
func makeVolumeDir(dirPath string) {
err := os.MkdirAll(dirPath, os.ModeDir|0700)
if err != nil {
log.Fatalf("Error creating %v: %v", dirPath, err)
}
func makeVolumeDir(dirPath string) error {
return os.MkdirAll(dirPath, os.ModeDir|0700)
}
func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) {
makeVolumeDir(sharedVolumePath)
makeVolumeDir(sharedSecretPath)
makeVolumeDir(sharedConfigPath)
func MakeFetcher(logger *zap.Logger, sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) {
fLogger := logger.Named("fetcher")
err := makeVolumeDir(sharedVolumePath)
if err != nil {
fLogger.Fatal("error creating shared volume directory", zap.Error(err), zap.String("directory", sharedVolumePath))
}
err = makeVolumeDir(sharedSecretPath)
if err != nil {
fLogger.Fatal("error creating shared secret directory", zap.Error(err), zap.String("directory", sharedSecretPath))
}
err = makeVolumeDir(sharedConfigPath)
if err != nil {
fLogger.Fatal("error creating shared config directory", zap.Error(err), zap.String("directory", sharedConfigPath))
}
fissionClient, kubeClient, _, err := crd.MakeFissionClient()
if err != nil {
return nil, err
return nil, errors.Wrap(err, "error making the fission / kube client")
}
return &Fetcher{
logger: fLogger,
sharedVolumePath: sharedVolumePath,
sharedSecretPath: sharedSecretPath,
sharedConfigPath: sharedConfigPath,
@@ -138,9 +148,7 @@ func writeSecretOrConfigMap(dataMap map[string][]byte, dirPath string) error {
writeFilePath := filepath.Join(dirPath, key)
err := ioutil.WriteFile(writeFilePath, val, 0600)
if err != nil {
e := fmt.Sprintf("Failed to write file %v: %v", writeFilePath, err)
log.Printf(e)
return errors.New(e)
return errors.Wrapf(err, "Failed to write file %s", writeFilePath)
}
}
return nil
@@ -160,38 +168,40 @@ func (fetcher *Fetcher) FetchHandler(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
defer func() {
elapsed := time.Since(startTime)
log.Printf("elapsed time in fetch request = %v", elapsed)
fetcher.logger.Info("fetch request done", zap.Duration("elapsed_time", elapsed))
}()
// parse request
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Printf("Error reading request body")
fetcher.logger.Error("error reading request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var req fission.FunctionFetchRequest
err = json.Unmarshal(body, &req)
if err != nil {
log.Printf("Error reading request body: %v", err)
fetcher.logger.Error("error parsing request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
code, err := fetcher.Fetch(r.Context(), req)
if err != nil {
fetcher.logger.Error("error fetching", zap.Error(err))
http.Error(w, err.Error(), code)
return
}
log.Printf("Checking secrets/cfgmaps")
fetcher.logger.Info("checking secrets/cfgmaps")
code, err = fetcher.FetchSecretsAndCfgMaps(req.Secrets, req.ConfigMaps)
if err != nil {
fetcher.logger.Error("error fetching secrets and config maps", zap.Error(err))
http.Error(w, err.Error(), code)
return
}
log.Printf("Completed fetch request")
fetcher.logger.Info("completed fetch request")
// all done
w.WriteHeader(http.StatusOK)
}
@@ -205,22 +215,21 @@ func (fetcher *Fetcher) SpecializeHandler(w http.ResponseWriter, r *http.Request
// parse request
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Printf("Error reading request body")
fetcher.logger.Error("error reading request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var req fission.FunctionSpecializeRequest
err = json.Unmarshal(body, &req)
if err != nil {
log.Printf("Error reading request body: %v", err)
fetcher.logger.Error("error parsing request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
//log.Printf("fetcher received fetch request and started downloading: %v", req)
err = fetcher.SpecializePod(r.Context(), req.FetchReq, req.LoadReq)
if err != nil {
fetcher.logger.Error("error specialing pod", zap.Error(err))
w.WriteHeader(http.StatusInternalServerError)
return
}
@@ -234,14 +243,16 @@ func (fetcher *Fetcher) SpecializeHandler(w http.ResponseWriter, r *http.Request
func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequest) (int, error) {
// check that the requested filename is not an empty string and error out if so
if len(req.Filename) == 0 {
e := fmt.Sprintf("Fetch request received for an empty file name, request: %v", req)
log.Printf(e)
return http.StatusBadRequest, errors.New(e)
e := "fetch request received for an empty file name"
fetcher.logger.Error(e, zap.Any("request", req))
return http.StatusBadRequest, errors.New(fmt.Sprintf("%s, request: %v", e, req))
}
// verify first if the file already exists.
if _, err := os.Stat(filepath.Join(fetcher.sharedVolumePath, req.Filename)); err == nil {
log.Printf("Requested file: %s already exists at %s. Skipping fetch", req.Filename, fetcher.sharedVolumePath)
fetcher.logger.Info("requested file already exists at shared volume - skipping fetch",
zap.String("requested_file", req.Filename),
zap.String("shared_volume_path", fetcher.sharedVolumePath))
return http.StatusOK, nil
}
@@ -252,17 +263,19 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
// fetch the file and save it to the tmp path
err := downloadUrl(ctx, fetcher.httpClient, req.Url, tmpPath)
if err != nil {
e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err)
log.Printf(e)
return http.StatusBadRequest, errors.New(e)
e := "failed to download url"
fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url))
return http.StatusBadRequest, errors.Wrapf(err, "%s: %s", e, req.Url)
}
} else {
// get pkg
pkg, err := fetcher.fissionClient.Packages(req.Package.Namespace).Get(req.Package.Name)
if err != nil {
e := fmt.Sprintf("Failed to get package: %v", err)
log.Printf(e)
return http.StatusInternalServerError, errors.New(e)
e := "failed to get package"
fetcher.logger.Error(e,
zap.String("package_name", req.Package.Name),
zap.String("package_namespace", req.Package.Namespace))
return http.StatusInternalServerError, errors.Wrap(err, e)
}
var archive *fission.Archive
@@ -274,9 +287,12 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
// we hit this "Get : unsupported protocol scheme "" error.
// it may be useful to the user if we can send a more meaningful error in such a scenario.
if pkg.Status.BuildStatus != fission.BuildStatusSucceeded && pkg.Status.BuildStatus != fission.BuildStatusNone {
e := fmt.Sprintf("Build status for the function's pkg : %s.%s is : %s, can't fetch deployment", pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Status.BuildStatus)
log.Printf(e)
return http.StatusInternalServerError, errors.New(e)
e := fmt.Sprintf("cannot fetch deployment: package build status was not %q", fission.BuildStatusSucceeded)
fetcher.logger.Error(e,
zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namespace", pkg.Metadata.Namespace),
zap.Any("package_build_status", pkg.Status.BuildStatus))
return http.StatusInternalServerError, errors.New(fmt.Sprintf("%s: pkg %s.%s has a status of %s", e, pkg.Metadata.Name, pkg.Metadata.Namespace, pkg.Status.BuildStatus))
}
archive = &pkg.Spec.Deployment
}
@@ -285,31 +301,31 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
// write pkg.Literal into tmpPath
err = ioutil.WriteFile(tmpPath, archive.Literal, 0600)
if err != nil {
e := fmt.Sprintf("Failed to write file %v: %v", tmpPath, err)
log.Printf(e)
return http.StatusInternalServerError, errors.New(e)
e := "failed to write file"
fetcher.logger.Error(e, zap.Error(err), zap.String("location", tmpPath))
return http.StatusInternalServerError, errors.Wrapf(err, "%s %s", e, tmpPath)
}
} else {
// download and verify
err := downloadUrl(ctx, fetcher.httpClient, archive.URL, tmpPath)
if err != nil {
e := fmt.Sprintf("Failed to download url %v: %v", req.Url, err)
log.Printf(e)
return http.StatusBadRequest, errors.New(e)
e := "failed to download url"
fetcher.logger.Error(e, zap.Error(err), zap.String("url", req.Url))
return http.StatusBadRequest, errors.Wrapf(err, "%s %s", e, req.Url)
}
checksum, err := getChecksum(tmpPath)
if err != nil {
e := fmt.Sprintf("Failed to get checksum: %v", err)
log.Printf(e)
return http.StatusBadRequest, errors.New(e)
e := "failed to get checksum"
fetcher.logger.Error(e, zap.Error(err))
return http.StatusBadRequest, errors.Wrap(err, e)
}
err = verifyChecksum(checksum, &archive.Checksum)
if err != nil {
e := fmt.Sprintf("Failed to verify checksum: %v", err)
log.Printf(e)
return http.StatusBadRequest, errors.New(e)
e := "failed to verify checksum"
fetcher.logger.Error(e, zap.Error(err))
return http.StatusBadRequest, errors.Wrap(err, e)
}
}
}
@@ -319,7 +335,10 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
tmpUnarchivePath := filepath.Join(fetcher.sharedVolumePath, uuid.NewV4().String())
err := fetcher.unarchive(tmpPath, tmpUnarchivePath)
if err != nil {
log.Println(err.Error())
fetcher.logger.Error("error unarchiving",
zap.Error(err),
zap.String("archive_location", tmpPath),
zap.String("target_location", tmpUnarchivePath))
return http.StatusInternalServerError, err
}
@@ -327,13 +346,17 @@ func (fetcher *Fetcher) Fetch(ctx context.Context, req fission.FunctionFetchRequ
}
// move tmp file to requested filename
err := fetcher.rename(tmpPath, filepath.Join(fetcher.sharedVolumePath, req.Filename))
renamePath := filepath.Join(fetcher.sharedVolumePath, req.Filename)
err := fetcher.rename(tmpPath, renamePath)
if err != nil {
log.Println(err.Error())
fetcher.logger.Error("error renaming file",
zap.Error(err),
zap.String("original_path", tmpPath),
zap.String("rename_path", renamePath))
return http.StatusInternalServerError, err
}
log.Printf("Successfully placed at %v", filepath.Join(fetcher.sharedVolumePath, req.Filename))
fetcher.logger.Info("successfully placed", zap.String("location", renamePath))
return http.StatusOK, nil
}
@@ -345,13 +368,17 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
data, err := fetcher.kubeClient.CoreV1().Secrets(secret.Namespace).Get(secret.Name, metav1.GetOptions{})
if err != nil {
e := fmt.Sprintf("Failed to get secret from kubeapi: %v", err)
log.Printf(e)
e := "error getting secret from kubeapi"
httpCode := http.StatusInternalServerError
if k8serr.IsNotFound(err) {
httpCode = http.StatusNotFound
e = "secret was not found in kubeapi"
}
fetcher.logger.Error(e,
zap.Error(err),
zap.String("secret_name", secret.Name),
zap.String("secret_namespace", secret.Namespace))
return httpCode, errors.New(e)
}
@@ -360,12 +387,21 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
secretDir := filepath.Join(fetcher.sharedSecretPath, secretPath)
err = os.MkdirAll(secretDir, os.ModeDir|0644)
if err != nil {
e := fmt.Sprintf("Failed to create directory %v: %v", secretDir, err)
log.Printf(e)
return http.StatusInternalServerError, errors.New(e)
e := "failed to create directory for secret"
fetcher.logger.Error(e,
zap.Error(err),
zap.String("directory", secretDir),
zap.String("secret_name", secret.Name),
zap.String("secret_namespace", secret.Namespace))
return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, secretDir)
}
err = writeSecretOrConfigMap(data.Data, secretDir)
if err != nil {
fetcher.logger.Error("failed to write secret to file location",
zap.Error(err),
zap.String("location", secretDir),
zap.String("secret_name", secret.Name),
zap.String("secret_namespace", secret.Namespace))
return http.StatusInternalServerError, err
}
}
@@ -376,13 +412,17 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
data, err := fetcher.kubeClient.CoreV1().ConfigMaps(config.Namespace).Get(config.Name, metav1.GetOptions{})
if err != nil {
e := fmt.Sprintf("Failed to get configmap from kubeapi: %v", err)
log.Printf(e)
e := "error getting configmap from kubeapi"
httpCode := http.StatusInternalServerError
if k8serr.IsNotFound(err) {
httpCode = http.StatusNotFound
e = "configmap was not found in kubeapi"
}
fetcher.logger.Error(e,
zap.Error(err),
zap.String("config_map_name", config.Name),
zap.String("config_map_namespace", config.Namespace))
return httpCode, errors.New(e)
}
@@ -391,9 +431,13 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
configDir := filepath.Join(fetcher.sharedConfigPath, configPath)
err = os.MkdirAll(configDir, os.ModeDir|0644)
if err != nil {
e := fmt.Sprintf("Failed to create directory %v: %v", configDir, err)
log.Printf(e)
return http.StatusInternalServerError, errors.New(e)
e := "failed to create directory for configmap"
fetcher.logger.Error(e,
zap.Error(err),
zap.String("directory", configDir),
zap.String("config_map_name", config.Name),
zap.String("config_map_namespace", config.Namespace))
return http.StatusInternalServerError, errors.Wrapf(err, "%s: %s", e, configDir)
}
configMap := make(map[string][]byte)
for key, val := range data.Data {
@@ -401,6 +445,11 @@ func (fetcher *Fetcher) FetchSecretsAndCfgMaps(secrets []fission.SecretReference
}
err = writeSecretOrConfigMap(configMap, configDir)
if err != nil {
fetcher.logger.Error("failed to write configmap to file location",
zap.Error(err),
zap.String("location", configDir),
zap.String("config_map_name", config.Name),
zap.String("config_map_namespace", config.Namespace))
return http.StatusInternalServerError, err
}
}
@@ -418,13 +467,13 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
startTime := time.Now()
defer func() {
elapsed := time.Since(startTime)
log.Printf("elapsed time in upload request = %v", elapsed)
fetcher.logger.Info("upload request done", zap.Duration("elapsed_time", elapsed))
}()
// parse request
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Printf("Error reading request body")
fetcher.logger.Error("error reading request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@@ -432,11 +481,11 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
var req fission.ArchiveUploadRequest
err = json.Unmarshal(body, &req)
if err != nil {
log.Printf("Error reading request body: %v", err)
fetcher.logger.Error("error parsing request body", zap.Error(err))
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
log.Printf("fetcher received upload request: %v", req)
fetcher.logger.Info("fetcher received upload request", zap.Any("request", req))
zipFilename := req.Filename + ".zip"
srcFilepath := filepath.Join(fetcher.sharedVolumePath, req.Filename)
@@ -445,37 +494,37 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
if req.ArchivePackage {
err = fetcher.archive(srcFilepath, dstFilepath)
if err != nil {
e := fmt.Sprintf("Error archiving zip file: %v", err)
log.Println(e)
http.Error(w, e, http.StatusInternalServerError)
e := "error archiving zip file"
fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath))
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return
}
} else {
err = os.Rename(srcFilepath, dstFilepath)
if err != nil {
e := fmt.Sprintf("Error renaming the archive: %v", err)
log.Println(e)
http.Error(w, e, http.StatusInternalServerError)
e := "error renaming the archive"
fetcher.logger.Error(e, zap.Error(err), zap.String("source", srcFilepath), zap.String("destination", dstFilepath))
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return
}
}
log.Println("Starting upload...")
fetcher.logger.Info("starting upload...")
ssClient := storageSvcClient.MakeClient(req.StorageSvcUrl)
fileID, err := ssClient.Upload(r.Context(), dstFilepath, nil)
if err != nil {
e := fmt.Sprintf("Error uploading zip file: %v", err)
log.Println(e)
http.Error(w, e, http.StatusInternalServerError)
e := "error uploading zip file"
fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath))
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return
}
sum, err := getChecksum(dstFilepath)
if err != nil {
e := fmt.Sprintf("Error calculating checksum of zip file: %v", err)
log.Println(e)
http.Error(w, e, http.StatusInternalServerError)
e := "error calculating checksum of zip file"
fetcher.logger.Error(e, zap.Error(err), zap.String("file", dstFilepath))
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return
}
@@ -486,13 +535,13 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
rBody, err := json.Marshal(resp)
if err != nil {
e := fmt.Sprintf("Error encoding upload response: %v", err)
log.Println(e)
http.Error(w, e, http.StatusInternalServerError)
e := "error encoding upload response"
fetcher.logger.Error(e, zap.Error(err))
http.Error(w, fmt.Sprintf("%s: %v", e, err), http.StatusInternalServerError)
return
}
log.Println("Completed upload request")
fetcher.logger.Info("completed upload request")
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(rBody)
@@ -501,7 +550,7 @@ func (fetcher *Fetcher) UploadHandler(w http.ResponseWriter, r *http.Request) {
func (fetcher *Fetcher) rename(src string, dst string) error {
err := os.Rename(src, dst)
if err != nil {
return errors.New(fmt.Sprintf("Failed to move file: %v", err))
return errors.Wrap(err, "failed to move file")
}
return nil
}
@@ -512,7 +561,7 @@ func (fetcher *Fetcher) archive(src string, dst string) error {
var files []string
target, err := os.Stat(src)
if err != nil {
return errors.New(fmt.Sprintf("Failed to zip file: %v", err))
return errors.Wrap(err, "failed to zip file")
}
if target.IsDir() {
// list all
@@ -530,7 +579,7 @@ func (fetcher *Fetcher) archive(src string, dst string) error {
func (fetcher *Fetcher) unarchive(src string, dst string) error {
err := archiver.Zip.Open(src, dst)
if err != nil {
return errors.New(fmt.Sprintf("Failed to unzip file: %v", err))
return errors.Wrap(err, "failed to unzip file")
}
return nil
}
@@ -539,17 +588,17 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func
startTime := time.Now()
defer func() {
elapsed := time.Since(startTime)
log.Printf("Elapsed time in fetch request = %v", elapsed)
fetcher.logger.Info("specialize request done", zap.Duration("elapsed_time", elapsed))
}()
_, err := fetcher.Fetch(ctx, fetchReq)
if err != nil {
return errors.Wrap(err, "Error fetching deploy package")
return errors.Wrap(err, "error fetching deploy package")
}
_, err = fetcher.FetchSecretsAndCfgMaps(fetchReq.Secrets, fetchReq.ConfigMaps)
if err != nil {
return errors.Wrap(err, "Error fetching secrets/configmaps")
return errors.Wrap(err, "error fetching secrets/configmaps")
}
// Specialize the pod
@@ -561,17 +610,19 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func
loadPayload, err := json.Marshal(loadReq)
if err != nil {
return errors.Wrap(err, "Error encoding load request")
return errors.Wrap(err, "error encoding load request")
}
if loadReq.EnvVersion >= 2 {
contentType = "application/json"
specializeURL = "http://localhost:8888/v2/specialize"
reader = bytes.NewReader(loadPayload)
fetcher.logger.Info("calling environment v2 specialization endpoint")
} else {
contentType = "text/plain"
specializeURL = "http://localhost:8888/specialize"
reader = bytes.NewReader([]byte{})
fetcher.logger.Info("calling environment v1 specialization endpoint")
}
for i := 0; i < maxRetries; i++ {
@@ -588,7 +639,7 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func
if netErr.Op == "dial" {
if i < maxRetries-1 {
time.Sleep(500 * time.Duration(2*i) * time.Millisecond)
log.Printf("Error connecting to pod (%v), retrying", netErr)
fetcher.logger.Error("error connecting to function environment pod for specialization request, retrying", zap.Error(netErr))
continue
}
}
@@ -599,8 +650,8 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq fission.Func
err = fission.MakeErrorFromHTTP(resp)
}
return errors.Wrap(err, "Error specializing function pod")
return errors.Wrap(err, "error specializing function pod")
}
return errors.Wrap(err, fmt.Sprintf("Error specializing function pod after %v times", maxRetries))
return errors.Wrapf(err, "error specializing function pod after %v times", maxRetries)
}
environments/go/server.go
+95 -76
View File
@@ -10,6 +10,7 @@ import (
"plugin"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/environments/go/context"
)
@@ -39,7 +40,7 @@ type (
var userFunc http.HandlerFunc
func loadPlugin(codePath, entrypoint string) (http.HandlerFunc, error) {
func loadPlugin(logger *zap.Logger, codePath, entrypoint string) (http.HandlerFunc, error) {
// if codepath's a directory, load the file inside it
info, err := os.Stat(codePath)
@@ -58,7 +59,7 @@ func loadPlugin(codePath, entrypoint string) (http.HandlerFunc, error) {
codePath = filepath.Join(codePath, fi.Name())
}
log.Printf("loading plugin from %v\n", codePath)
logger.Info("loading plugin", zap.String("location", codePath))
p, err := plugin.Open(codePath)
if err != nil {
return nil, errors.Wrap(err, "error loading plugin")
@@ -85,87 +86,99 @@ func loadPlugin(codePath, entrypoint string) (http.HandlerFunc, error) {
}
}
func specializeHandler(w http.ResponseWriter, r *http.Request) {
if userFunc != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Not a generic container"))
return
}
_, err := os.Stat(CODE_PATH)
if err != nil {
if os.IsNotExist(err) {
w.WriteHeader(http.StatusNotFound)
log.Printf(err.Error())
w.Write([]byte(CODE_PATH + ": not found"))
return
} else {
err = errors.Wrap(err, "unknown error")
log.Printf(err.Error())
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
func specializeHandler(logger *zap.Logger) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
if userFunc != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Not a generic container"))
return
}
}
log.Println("Specializing ...")
userFunc, err = loadPlugin(CODE_PATH, "Handler")
if err != nil {
err = errors.Wrap(err, "error specializing function")
log.Printf(err.Error())
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
_, err := os.Stat(CODE_PATH)
if err != nil {
if os.IsNotExist(err) {
w.WriteHeader(http.StatusNotFound)
logger.Error("code path does not exist",
zap.Error(err),
zap.String("code_path", CODE_PATH))
w.Write([]byte(CODE_PATH + ": not found"))
return
} else {
logger.Error("unknown error looking for code path",
zap.Error(err),
zap.String("code_path", CODE_PATH))
err = errors.Wrap(err, "unknown error")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
}
logger.Info("specializing ...")
userFunc, err = loadPlugin(logger, CODE_PATH, "Handler")
if err != nil {
e := "error specializing function"
logger.Error(e, zap.Error(err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(errors.Wrap(err, e).Error()))
return
}
logger.Info("done")
}
log.Println("Done")
}
func specializeHandlerV2(w http.ResponseWriter, r *http.Request) {
if userFunc != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Not a generic container"))
return
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Println(err.Error())
w.WriteHeader(http.StatusInternalServerError)
return
}
var loadreq FunctionLoadRequest
err = json.Unmarshal(body, &loadreq)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
_, err = os.Stat(loadreq.FilePath)
if err != nil {
if os.IsNotExist(err) {
w.WriteHeader(http.StatusNotFound)
log.Printf(err.Error())
w.Write([]byte(CODE_PATH + ": not found"))
return
} else {
err = errors.Wrap(err, "unknown error")
log.Printf(err.Error())
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
func specializeHandlerV2(logger *zap.Logger) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
if userFunc != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Not a generic container"))
return
}
}
log.Println("Specializing ...")
userFunc, err = loadPlugin(loadreq.FilePath, loadreq.FunctionName)
if err != nil {
err = errors.Wrap(err, "error specializing function")
log.Printf(err.Error())
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
body, err := ioutil.ReadAll(r.Body)
if err != nil {
logger.Error("error reading request body", zap.Error(err))
w.WriteHeader(http.StatusInternalServerError)
return
}
var loadreq FunctionLoadRequest
err = json.Unmarshal(body, &loadreq)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
_, err = os.Stat(loadreq.FilePath)
if err != nil {
if os.IsNotExist(err) {
logger.Error("code path does not exist",
zap.Error(err),
zap.String("code_path", CODE_PATH))
w.WriteHeader(http.StatusNotFound)
w.Write([]byte(CODE_PATH + ": not found"))
return
} else {
logger.Error("unknown error looking for code path",
zap.Error(err),
zap.String("code_path", CODE_PATH))
err = errors.Wrap(err, "unknown error")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
}
logger.Info("specializing ...")
userFunc, err = loadPlugin(logger, loadreq.FilePath, loadreq.FunctionName)
if err != nil {
e := "error specializing function"
logger.Error(e, zap.Error(err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(errors.Wrap(err, e).Error()))
return
}
logger.Info("done")
}
log.Println("Done")
}
func readinessProbeHandler(w http.ResponseWriter, r *http.Request) {
@@ -173,9 +186,15 @@ func readinessProbeHandler(w http.ResponseWriter, r *http.Request) {
}
func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
http.HandleFunc("/healthz", readinessProbeHandler)
http.HandleFunc("/specialize", specializeHandler)
http.HandleFunc("/v2/specialize", specializeHandlerV2)
http.HandleFunc("/specialize", specializeHandler(logger.Named("specialize_handler")))
http.HandleFunc("/v2/specialize", specializeHandlerV2(logger.Named("specialize_v2_handler")))
// Generic route -- all http requests go to the user function.
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
@@ -187,6 +206,6 @@ func main() {
userFunc(w, r)
})
log.Println("Listening on 8888 ...")
logger.Info("listening on 8888 ...")
http.ListenAndServe(":8888", nil)
}
executor/api.go
+20 -8
View File
@@ -21,12 +21,12 @@ import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strings"
"github.com/gorilla/mux"
"go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
@@ -50,7 +50,10 @@ func (executor *Executor) getServiceForFunctionApi(w http.ResponseWriter, r *htt
serviceName, err := executor.getServiceForFunction(r.Context(), &m)
if err != nil {
code, msg := fission.GetHTTPError(err)
log.Printf("Error: %v: %v", code, msg)
executor.logger.Error("error getting service for function",
zap.Error(err),
zap.String("function", m.Name),
zap.String("fission_http_error", msg))
http.Error(w, msg, code)
return
}
@@ -69,14 +72,19 @@ func (executor *Executor) getServiceForFunctionApi(w http.ResponseWriter, r *htt
// invalidates the cache entry if the pod address was cached.
func (executor *Executor) getServiceForFunction(ctx context.Context, m *metav1.ObjectMeta) (string, error) {
// Check function -> svc cache
log.Printf("[%v] Checking for cached function service", m.Name)
executor.logger.Info("checking for cached function service",
zap.String("function_name", m.Name),
zap.String("function_namespace", m.Namespace))
fsvc, err := executor.fsCache.GetByFunction(m)
if err == nil {
if executor.isValidAddress(fsvc) {
// Cached, return svc address
return fsvc.Address, nil
} else {
log.Printf("[%v] Deleting cache entry for invalid address : %s", m.Name, fsvc.Address)
executor.logger.Info("deleting cache entry for invalid address",
zap.String("function_name", m.Name),
zap.String("function_namespace", m.Namespace),
zap.String("address", fsvc.Address))
executor.fsCache.DeleteEntry(fsvc)
}
}
@@ -98,6 +106,7 @@ func (executor *Executor) getServiceForFunction(ctx context.Context, m *metav1.O
func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
executor.logger.Error("failed to read tap service request", zap.Error(err))
http.Error(w, "Failed to read request", http.StatusInternalServerError)
return
}
@@ -106,7 +115,10 @@ func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) {
err = executor.fsCache.TouchByAddress(svcHost)
if err != nil {
log.Printf("funcSvc tap error: %v", err)
executor.logger.Error("error tapping function service",
zap.Error(err),
zap.String("service", svcName),
zap.String("host", svcHost))
http.Error(w, "Not found", http.StatusNotFound)
return
}
@@ -123,15 +135,15 @@ func (executor *Executor) Serve(port int) {
r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST")
r.HandleFunc("/healthz", executor.healthHandler).Methods("GET")
address := fmt.Sprintf(":%v", port)
log.Printf("starting executor at port %v", port)
executor.logger.Info("starting executor", zap.Int("port", port))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
executor.ndm.Run(ctx)
executor.gpm.Run(ctx)
r.Use(fission.LoggingMiddleware)
r.Use(fission.LoggingMiddleware(executor.logger))
err := http.ListenAndServe(address, &ochttp.Handler{
Handler: r,
// Propagation: &b3.HTTPFormat{},
})
log.Fatal(err)
executor.logger.Fatal("done listening", zap.Error(err))
}
executor/client/client.go
+10 -8
View File
@@ -21,13 +21,14 @@ import (
"context"
"encoding/json"
"io/ioutil"
"log"
"net/http"
"net/url"
"strings"
"time"
"github.com/pkg/errors"
"go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
"golang.org/x/net/context/ctxhttp"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,14 +36,16 @@ import (
)
type Client struct {
logger *zap.Logger
executorUrl string
tappedByUrl map[string]bool
requestChan chan string
httpClient *http.Client
}
func MakeClient(executorUrl string) *Client {
func MakeClient(logger *zap.Logger, executorUrl string) *Client {
c := &Client{
logger: logger.Named("executor_client"),
executorUrl: strings.TrimSuffix(executorUrl, "/"),
tappedByUrl: make(map[string]bool),
requestChan: make(chan string),
@@ -59,12 +62,12 @@ func (c *Client) GetServiceForFunction(ctx context.Context, metadata *metav1.Obj
body, err := json.Marshal(metadata)
if err != nil {
return "", err
return "", errors.Wrap(err, "could not marshal request body for getting service for function")
}
resp, err := ctxhttp.Post(ctx, c.httpClient, executorUrl, "application/json", bytes.NewReader(body))
if err != nil {
return "", err
return "", errors.Wrap(err, "error posting to getting service for function")
}
defer resp.Body.Close()
@@ -74,8 +77,7 @@ func (c *Client) GetServiceForFunction(ctx context.Context, metadata *metav1.Obj
svcName, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Returning from ioutil read body")
return "", err
return "", errors.Wrap(err, "error reading response body from getting service for function")
}
return string(svcName), nil
@@ -95,10 +97,10 @@ func (c *Client) service() {
for u := range urls {
err := c._tapService(u)
if err != nil {
log.Printf("Error tapping function service address %v: %v", u, err)
c.logger.Error("error tapping function service address", zap.Error(err), zap.String("address", u))
}
}
log.Printf("Tapped %v services in batch", len(urls))
c.logger.Info("tapped services in batch", zap.Int("service_count", len(urls)))
}()
}
}
executor/executor.go
+33 -18
View File
@@ -19,7 +19,6 @@ package executor
import (
"context"
"fmt"
"log"
"net/http"
"runtime/debug"
"strings"
@@ -29,6 +28,7 @@ import (
"github.com/dchest/uniuri"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
@@ -41,6 +41,8 @@ import (
type (
Executor struct {
logger *zap.Logger
gpm *poolmgr.GenericPoolManager
ndm *newdeploy.NewDeploy
@@ -62,8 +64,9 @@ type (
}
)
func MakeExecutor(gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor {
func MakeExecutor(logger *zap.Logger, gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor {
executor := &Executor{
logger: logger.Named("executor"),
gpm: gpm,
ndm: ndm,
fissionClient: fissionClient,
@@ -111,7 +114,8 @@ func (executor *Executor) serveCreateFuncServices() {
} else {
// There's an existing request for this function, wait for it to finish
go func() {
log.Printf("Waiting for concurrent request for the same function: %v", m)
executor.logger.Info("waiting for concurrent request for the same function",
zap.Any("function", m))
wg.Wait()
// get the function service from the cache
@@ -121,7 +125,9 @@ func (executor *Executor) serveCreateFuncServices() {
// It normally happened if there are multiple requests are
// waiting for the same function and executor failed to cre-
// ate service for function.
err = errors.Wrap(err, fmt.Sprintf("Error getting service for function %v in namespace %v", m.Name, m.Namespace))
err = errors.Wrapf(err, "error getting service for function",
zap.String("function_name", m.Name),
zap.String("function_namespace", m.Namespace))
req.respChan <- &createFuncServiceResponse{
funcSvc: fsvc,
err: err,
@@ -140,7 +146,9 @@ func (executor *Executor) getFunctionExecutorType(meta *metav1.ObjectMeta) (fiss
}
func (executor *Executor) createServiceForFunction(ctx context.Context, meta *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
log.Printf("[%v] No cached function service found, creating one", meta.Name)
executor.logger.Info("no cached function service found, creating one",
zap.String("function_name", meta.Name),
zap.String("function_namespace", meta.Namespace))
executorType, err := executor.getFunctionExecutorType(meta)
if err != nil {
@@ -158,8 +166,12 @@ func (executor *Executor) createServiceForFunction(ctx context.Context, meta *me
}
if fsvcErr != nil {
fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%v] Error creating service for function", meta.Name))
log.Print(fsvcErr)
e := "error creating service for function"
executor.logger.Error(e,
zap.Error(fsvcErr),
zap.String("function_name", meta.Name),
zap.String("function_namespace", meta.Namespace))
fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%s] %s", meta.Name, e))
} else if fsvc != nil {
_, err = executor.fsCache.Add(*fsvc)
if err != nil {
@@ -185,16 +197,18 @@ func dumpStackTrace() {
debug.PrintStack()
}
func serveMetric() {
func serveMetric(logger *zap.Logger) {
// Expose the registered metrics via HTTP.
metricAddr := ":8080"
http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(metricAddr, nil))
err := http.ListenAndServe(metricAddr, nil)
logger.Fatal("done listening on metrics endpoint", zap.Error(err))
}
// StartExecutor Starts executor and the executor components such as Poolmgr,
// deploymgr and potential future executor types
func StartExecutor(fissionNamespace string, functionNamespace string, envBuilderNamespace string, port int) error {
func StartExecutor(logger *zap.Logger, fissionNamespace string, functionNamespace string, envBuilderNamespace string, port int) error {
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
@@ -202,33 +216,34 @@ func StartExecutor(fissionNamespace string, functionNamespace string, envBuilder
err = fissionClient.WaitForCRDs()
if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err)
return errors.Wrap(err, "error waiting for CRDs")
}
restClient := fissionClient.GetCrdClient()
if err != nil {
log.Printf("Failed to get kubernetes client: %v", err)
return err
return errors.Wrap(err, "failed to get kubernetes client")
}
fsCache := fscache.MakeFunctionServiceCache()
fsCache := fscache.MakeFunctionServiceCache(logger)
poolID := strings.ToLower(uniuri.NewLen(8))
reaper.CleanupOldExecutorObjects(kubernetesClient, poolID)
go reaper.CleanupRoleBindings(kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
reaper.CleanupOldExecutorObjects(logger, kubernetesClient, poolID)
go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
gpm := poolmgr.MakeGenericPoolManager(
logger,
fissionClient, kubernetesClient,
functionNamespace, poolID)
ndm := newdeploy.MakeNewDeploy(
logger,
fissionClient, kubernetesClient, restClient,
functionNamespace, poolID)
api := MakeExecutor(gpm, ndm, fissionClient, fsCache)
api := MakeExecutor(logger, gpm, ndm, fissionClient, fsCache)
go api.Serve(port)
go serveMetric()
go serveMetric(logger)
return nil
}
executor/executor_test.go
+13 -3
View File
@@ -33,6 +33,7 @@ import (
"testing"
"time"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
@@ -44,6 +45,12 @@ import (
"github.com/fission/fission/executor/client"
)
func panicIf(err error) {
if err != nil {
log.Panicf("Error: %v", err)
}
}
// return the number of pods in the given namespace matching the given labels
func countPods(kubeClient *kubernetes.Clientset, ns string, labelz map[string]string) int {
pods, err := kubeClient.Pods(ns).List(metav1.ListOptions{
@@ -134,8 +141,11 @@ func TestExecutor(t *testing.T) {
createTestNamespace(kubeClient, functionNs)
defer kubeClient.Namespaces().Delete(functionNs, nil)
logger, err := zap.NewDevelopment()
panicIf(err)
// make sure CRD types exist on cluster
err = crd.EnsureFissionCRDs(apiExtClient)
err = crd.EnsureFissionCRDs(logger, apiExtClient)
if err != nil {
log.Panicf("failed to ensure crds: %v", err)
}
@@ -165,13 +175,13 @@ func TestExecutor(t *testing.T) {
// create poolmgr
port := 9999
err = StartExecutor(fissionNs, functionNs, port)
err = StartExecutor(logger, fissionNs, functionNs, "fission-builder", port)
if err != nil {
log.Panicf("failed to start poolmgr: %v", err)
}
// connect poolmgr client
poolmgrClient := client.MakeClient(fmt.Sprintf("http://localhost:%v", port))
poolmgrClient := client.MakeClient(logger, fmt.Sprintf("http://localhost:%v", port))
// Wait for pool to be created (we don't actually need to do
// this, since the API should do the right thing in any case).
executor/fscache/functionServiceCache.go
+16 -12
View File
@@ -17,9 +17,11 @@ limitations under the License.
package fscache
import (
"log"
"fmt"
"time"
"github.com/pkg/errors"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
@@ -57,6 +59,7 @@ type (
}
FunctionServiceCache struct {
logger *zap.Logger
byFunction *cache.Cache // function-key -> funcSvc : map[string]*funcSvc
byAddress *cache.Cache // address -> function : map[string]metav1.ObjectMeta
byFunctionUID *cache.Cache // function uid -> function : map[string]metav1.ObjectMeta
@@ -91,8 +94,9 @@ func IsNameExistError(err error) bool {
return false
}
func MakeFunctionServiceCache() *FunctionServiceCache {
func MakeFunctionServiceCache(logger *zap.Logger) *FunctionServiceCache {
fsc := &FunctionServiceCache{
logger: logger.Named("function_service_cache"),
byFunction: cache.MakeCache(0, 0),
byAddress: cache.MakeCache(0, 0),
byFunctionUID: cache.MakeCache(0, 0),
@@ -122,14 +126,16 @@ func (fsc *FunctionServiceCache) service() {
}
resp.objects = funcObjects
case LOG:
fsc.logger.Info("dumping function service cache")
funcCopy := fsc.byFunction.Copy()
log.Printf("Cache has %v entries", len(funcCopy))
info := []string{}
for key, fsvcI := range funcCopy {
fsvc := fsvcI.(*FuncSvc)
for _, kubeObj := range fsvc.KubernetesObjects {
log.Printf("%v\t%v\t%v", key, kubeObj.Kind, kubeObj.Name)
info = append(info, fmt.Sprintf("%v\t%v\t%v", key, kubeObj.Kind, kubeObj.Name))
}
}
fsc.logger.Info("function service cache", zap.Int("item_count", len(funcCopy)), zap.Strings("cache", info))
}
req.responseChannel <- resp
}
@@ -196,9 +202,8 @@ func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {
if err != nil {
if IsNameExistError(err) {
err = nil
}
if err != nil {
log.Printf("error caching fsvc: %v", err)
} else {
err = errors.Wrap(err, "error caching fsvc")
}
return nil, err
}
@@ -209,9 +214,8 @@ func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {
if err != nil {
if IsNameExistError(err) {
err = nil
}
if err != nil {
log.Printf("error caching fsvc by function uid: %v", err)
} else {
err = errors.Wrap(err, "error caching fsvc by function uid")
}
return nil, err
}
@@ -278,12 +282,12 @@ func (fsc *FunctionServiceCache) ListOld(age time.Duration) ([]*FuncSvc, error)
}
func (fsc *FunctionServiceCache) Log() {
log.Printf("--- FunctionService Cache Contents")
fsc.logger.Info("--- FunctionService Cache Contents")
responseChannel := make(chan *fscResponse)
fsc.requestChannel <- &fscRequest{
requestType: LOG,
responseChannel: responseChannel,
}
<-responseChannel
log.Printf("--- FunctionService Cache Contents End")
fsc.logger.Info("--- FunctionService Cache Contents End")
}
executor/fscache/functionServiceCache_test.go
+11 -1
View File
@@ -5,6 +5,7 @@ import (
"testing"
"time"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -12,8 +13,17 @@ import (
"github.com/fission/fission/crd"
)
func panicIf(err error) {
if err != nil {
log.Panicf("Error: %v", err)
}
}
func TestFunctionServiceCache(t *testing.T) {
fsc := MakeFunctionServiceCache()
logger, err := zap.NewDevelopment()
panicIf(err)
fsc := MakeFunctionServiceCache(logger)
if fsc == nil {
log.Panicf("error creating cache")
}
executor/newdeploy/newdeploy.go
+42 -15
View File
@@ -20,11 +20,11 @@ import (
"encoding/json"
"errors"
"fmt"
"log"
"path/filepath"
"time"
"github.com/hashicorp/go-multierror"
"go.uber.org/zap"
asv1 "k8s.io/api/autoscaling/v1"
apiv1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1"
@@ -59,10 +59,9 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro
existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Get(deployName, metav1.GetOptions{})
if err == nil {
if waitForDeploy {
err = scaleDeployment(deploy.kubernetesClient,
existingDepl.Namespace, existingDepl.Name, minScale)
err = deploy.scaleDeployment(existingDepl.Namespace, existingDepl.Name, minScale)
if err != nil {
log.Printf("Error scaling up deployment for function %v: %v", fn.Metadata.Name, err)
deploy.logger.Error("error scaling up function deployment", zap.Error(err), zap.String("function", fn.Metadata.Name))
return nil, err
}
@@ -86,7 +85,11 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro
depl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Create(deployment)
if err != nil {
log.Printf("Error while creating deployment: %v", err)
deploy.logger.Error("error while creating function deployment",
zap.Error(err),
zap.String("function", fn.Metadata.Name),
zap.String("deployment_name", deployName),
zap.String("deployment_namespace", deployNamespace))
return nil, err
}
@@ -105,25 +108,40 @@ func (deploy *NewDeploy) setupRBACObjs(deployNamespace string, fn *crd.Function)
// create fetcher SA in this ns, if not already created
_, err := fission.SetupSA(deploy.kubernetesClient, fission.FissionFetcherSA, deployNamespace)
if err != nil {
log.Printf("Error : %v creating %s in ns : %s for function: %s.%s", err, fission.FissionFetcherSA, deployNamespace, fn.Metadata.Name, fn.Metadata.Namespace)
deploy.logger.Error("error creating fission fetcher service account for function",
zap.Error(err),
zap.String("service_account_name", fission.FissionFetcherSA),
zap.String("service_account_namespace", deployNamespace),
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
return err
}
// create a cluster role binding for the fetcher SA, if not already created, granting access to do a get on packages in any ns
err = fission.SetupRoleBinding(deploy.kubernetesClient, fission.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace)
err = fission.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, fission.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace)
if err != nil {
log.Printf("Error : %v creating %s RoleBinding for function: %s.%s", err, fission.PackageGetterRB, fn.Metadata.Name, fn.Metadata.Namespace)
deploy.logger.Error("error creating role binding for function",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
return err
}
// create rolebinding in function namespace for fetcherSA.envNamespace to be able to get secrets and configmaps
err = fission.SetupRoleBinding(deploy.kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace)
err = fission.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, deployNamespace)
if err != nil {
log.Printf("Error : %v creating %s RoleBinding for function %s.%s", err, fission.SecretConfigMapGetterRB, fn.Metadata.Name, fn.Metadata.Namespace)
deploy.logger.Error("error creating role binding for function",
zap.Error(err),
zap.String("role_binding", fission.SecretConfigMapGetterRB),
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
return err
}
log.Printf("Set up all RBAC objects for function : %s.%s", fn.Metadata.Name, fn.Metadata.Namespace)
deploy.logger.Info("set up all RBAC objects for function",
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
return nil
}
@@ -188,7 +206,7 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen
fetcherResources, err := util.GetFetcherResources()
if err != nil {
log.Printf("Error while parsing fetcher resources: %v", err)
deploy.logger.Error("error while parsing fetcher resources", zap.Error(err))
return nil, err
}
@@ -515,19 +533,28 @@ func (deploy *NewDeploy) cleanupNewdeploy(ns string, name string) error {
err := deploy.deleteSvc(ns, name)
if err != nil {
log.Printf("Error deleting service for newdeploy function %v in namespace %v, error: %v", name, ns, err)
deploy.logger.Error("error deleting service for newdeploy function",
zap.Error(err),
zap.String("function_name", name),
zap.String("function_namespace", ns))
multierror.Append(multierr, err)
}
err = deploy.deleteHpa(ns, name)
if err != nil {
log.Printf("Error deleting HPA for newdeploy function %v in namespace %v, error: %v", name, ns, err)
deploy.logger.Error("error deleting service for newdeploy function",
zap.Error(err),
zap.String("function_name", name),
zap.String("function_namespace", ns))
multierror.Append(multierr, err)
}
err = deploy.deleteDeployment(ns, name)
if err != nil {
log.Printf("Error deleting deployment for newdeploy function %v in namespace %v, error: %v", name, ns, err)
deploy.logger.Error("error deleting deployment for newdeploy function",
zap.Error(err),
zap.String("function_name", name),
zap.String("function_namespace", ns))
multierror.Append(multierr, err)
}
return multierr.ErrorOrNil()
executor/newdeploy/newdeploymgr.go
+63 -43
View File
@@ -19,7 +19,6 @@ package newdeploy
import (
"context"
"fmt"
"log"
"os"
"strconv"
"strings"
@@ -29,6 +28,7 @@ import (
"github.com/fission/fission/throttler"
multierror "github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1"
k8sErrs "k8s.io/apimachinery/pkg/api/errors"
@@ -46,6 +46,8 @@ import (
type (
NewDeploy struct {
logger *zap.Logger
kubernetesClient *kubernetes.Clientset
fissionClient *crd.FissionClient
crdClient *rest.RESTClient
@@ -72,6 +74,7 @@ type (
)
func MakeNewDeploy(
logger *zap.Logger,
fissionClient *crd.FissionClient,
kubernetesClient *kubernetes.Clientset,
crdClient *rest.RESTClient,
@@ -79,7 +82,7 @@ func MakeNewDeploy(
instanceID string,
) *NewDeploy {
log.Printf("Creating NewDeploy ExecutorType")
logger.Info("creating NewDeploy ExecutorType")
fetcherImg := os.Getenv("FETCHER_IMAGE")
if len(fetcherImg) == 0 {
@@ -92,19 +95,21 @@ func MakeNewDeploy(
if len(os.Getenv("ENABLE_ISTIO")) > 0 {
istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
if err != nil {
log.Println("Failed to parse ENABLE_ISTIO")
logger.Info("failed to parse 'ENABLE_ISTIO'")
}
enableIstio = istio
}
nd := &NewDeploy{
logger: logger.Named("new_deploy"),
fissionClient: fissionClient,
kubernetesClient: kubernetesClient,
crdClient: crdClient,
instanceID: instanceID,
namespace: namespace,
fsCache: fscache.MakeFunctionServiceCache(),
fsCache: fscache.MakeFunctionServiceCache(logger),
throttler: throttler.MakeThrottler(1 * time.Minute),
fetcherImg: fetcherImg,
@@ -142,14 +147,18 @@ func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controll
fn := obj.(*crd.Function)
_, err := deploy.createFunction(fn, true)
if err != nil {
log.Printf("Error eager creating function: %v", err)
deploy.logger.Error("error eager creating function",
zap.Error(err),
zap.Any("function", fn))
}
},
DeleteFunc: func(obj interface{}) {
fn := obj.(*crd.Function)
err := deploy.deleteFunction(fn)
if err != nil {
log.Printf("Error deleing the function: %v", err)
deploy.logger.Error("error deleting function",
zap.Error(err),
zap.Any("function", fn))
}
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
@@ -157,7 +166,10 @@ func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controll
newFn := newObj.(*crd.Function)
err := deploy.updateFunction(oldFn, newFn)
if err != nil {
log.Printf("Error deleing the function: %v", err)
deploy.logger.Error("error updating function",
zap.Error(err),
zap.Any("old_function", oldFn),
zap.Any("new_function", newFn))
}
},
})
@@ -186,7 +198,7 @@ func (deploy *NewDeploy) createFunction(fn *crd.Function, firstcreate bool) (*fs
fsvc, ok := fsvcObj.(*fscache.FuncSvc)
if !ok {
log.Panic("Receive unknown object, expect pointer of function service object")
deploy.logger.Panic("receive unknown object while creating function - expected pointer of function service object")
}
return fsvc, err
@@ -198,7 +210,7 @@ func (deploy *NewDeploy) deleteFunction(fn *crd.Function) error {
}
err := deploy.fnDelete(fn)
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error deleting kubernetes objects of function %v", fn.Metadata))
err = errors.Wrapf(err, "error deleting kubernetes objects of function %v", fn.Metadata)
}
return err
}
@@ -235,22 +247,23 @@ func (deploy *NewDeploy) fnCreate(fn *crd.Function, firstcreate bool) (*fscache.
// then deployment) to take advantage of waiting time.
svc, err := deploy.createOrGetSvc(deployLabels, objName, ns)
if err != nil {
log.Printf("Error creating the service %v: %v", objName, err)
deploy.logger.Error("error creating service", zap.Error(err), zap.String("service", objName))
go deploy.cleanupNewdeploy(ns, objName)
return nil, errors.Wrap(err, fmt.Sprintf("error creating service %v", objName))
return nil, errors.Wrapf(err, "error creating service %v", objName)
}
svcAddress := fmt.Sprintf("%v.%v", svc.Name, svc.Namespace)
depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels, ns, firstcreate)
if err != nil {
log.Printf("Error creating the deployment %v: %v", objName, err)
deploy.logger.Error("error creating deployment", zap.Error(err), zap.String("deployment", objName))
go deploy.cleanupNewdeploy(ns, objName)
return nil, errors.Wrap(err, fmt.Sprintf("error creating deployment %v", objName))
return nil, errors.Wrapf(err, "error creating deployment %v", objName)
}
hpa, err := deploy.createOrGetHpa(objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl)
if err != nil {
deploy.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName))
go deploy.cleanupNewdeploy(ns, objName)
return nil, errors.Wrap(err, fmt.Sprintf("error creating the HPA %v:", objName))
return nil, errors.Wrapf(err, "error creating the HPA %v", objName)
}
kubeObjRefs := []apiv1.ObjectReference{
@@ -292,7 +305,7 @@ func (deploy *NewDeploy) fnCreate(fn *crd.Function, firstcreate bool) (*fscache.
_, err = deploy.fsCache.Add(*fsvc)
if err != nil {
log.Printf("Error adding the function to cache: %v", err)
deploy.logger.Error("error adding function to cache", zap.Error(err), zap.Any("function", fsvc.Function))
return fsvc, err
}
return fsvc, nil
@@ -313,7 +326,8 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
// Executor type is no longer New Deployment
if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy &&
oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy {
log.Printf("function does not use new deployment executor anymore, deleting resources: %v", newFn)
deploy.logger.Info("function does not use new deployment executor anymore, deleting resources",
zap.Any("function", newFn))
// IMP - pass the oldFn, as the new/modified function is not in cache
return deploy.deleteFunction(oldFn)
}
@@ -321,17 +335,19 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
// Executor type changed to New Deployment from something else
if oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy &&
newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy {
log.Printf("function type changed to new deployment, creating resources: %v %v", oldFn.Metadata, newFn.Metadata)
deploy.logger.Info("function type changed to new deployment, creating resources",
zap.Any("old_function", oldFn.Metadata),
zap.Any("new_function", newFn.Metadata))
_, err := deploy.createFunction(newFn, true)
if err != nil {
updateStatus(oldFn, err, "error changing the function's type to newdeploy")
deploy.updateStatus(oldFn, err, "error changing the function's type to newdeploy")
}
return err
}
fsvc, err := deploy.fsCache.GetByFunctionUID(newFn.Metadata.UID)
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error updating function due to unable to find function service cache: %v", oldFn))
err = errors.Wrapf(err, "error updating function due to unable to find function service cache: %v", oldFn)
return err
}
@@ -349,7 +365,7 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
hpa, err := deploy.getHpa(ns, fnObjName)
if err != nil {
updateStatus(oldFn, err, "error getting HPA while updating function")
deploy.updateStatus(oldFn, err, "error getting HPA while updating function")
return err
}
@@ -375,7 +391,7 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
if hpaChanged {
err := deploy.updateHpa(hpa)
if err != nil {
updateStatus(oldFn, err, "error updating HPA while updating function")
deploy.updateStatus(oldFn, err, "error updating HPA while updating function")
return err
}
}
@@ -413,15 +429,15 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
env, err := deploy.fissionClient.Environments(newFn.Spec.Environment.Namespace).
Get(newFn.Spec.Environment.Name)
if err != nil {
updateStatus(oldFn, err, "failed to get environment while updating function")
deploy.updateStatus(oldFn, err, "failed to get environment while updating function")
return err
}
deployLabels := deploy.getDeployLabels(oldFn, env)
log.Printf("updating %v deployment due to function %v update", fnObjName, newFn.Metadata.Name)
deploy.logger.Info("updating deployment due to function update", zap.String("deployment", fnObjName), zap.String("function", newFn.Metadata.Name))
newDeployment, err := deploy.getDeploymentSpec(newFn, env, fnObjName, deployLabels)
if err != nil {
updateStatus(oldFn, err, "failed to get new deployment spec while updating function")
deploy.updateStatus(oldFn, err, "failed to get new deployment spec while updating function")
return err
}
@@ -434,7 +450,7 @@ func (deploy *NewDeploy) updateFunction(oldFn *crd.Function, newFn *crd.Function
err = deploy.updateDeployment(newDeployment, ns)
if err != nil {
updateStatus(oldFn, err, "failed to update deployment while updating function")
deploy.updateStatus(oldFn, err, "failed to update deployment while updating function")
return err
}
}
@@ -506,13 +522,13 @@ func (deploy *NewDeploy) updateKubeObjRefRV(fsvc *fscache.FuncSvc, objKind strin
}
}
fsvc.KubernetesObjects = kubeObjs
return errors.New(fmt.Sprintf("error finding kubernetes object reference with kind: %v", objKind))
return fmt.Errorf("error finding kubernetes object reference with kind: %v", objKind)
}
// updateStatus is a function which updates status of update.
// Current implementation only logs messages, in future it will update function status
func updateStatus(fn *crd.Function, err error, message string) {
log.Println(message, fn, err)
func (deploy *NewDeploy) updateStatus(fn *crd.Function, err error, message string) {
deploy.logger.Info("function status update", zap.Error(err), zap.Any("function", fn), zap.String("message", message))
}
// IsValid does a get on the service address to ensure it's a valid service, then
@@ -526,20 +542,20 @@ func (deploy *NewDeploy) IsValid(fsvc *fscache.FuncSvc) bool {
_, err := deploy.kubernetesClient.CoreV1().Services(service[1]).Get(service[0], metav1.GetOptions{})
if err != nil {
log.Printf("Error validating service address for function %v: %v", fsvc.Function.Name, err)
deploy.logger.Error("error validating function service address", zap.String("function", fsvc.Function.Name), zap.Error(err))
return false
}
deployObj := getDeploymentObj(fsvc.KubernetesObjects)
if deployObj == nil {
log.Printf("Deployment obj for function %v does not exist", fsvc.Function.Name)
deploy.logger.Error("deployment obj for function does not exist", zap.String("function", fsvc.Function.Name))
return false
}
currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
if err != nil {
log.Printf("Error validating deployment for function %v: %v", fsvc.Function.Name, err)
deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
return false
}
@@ -560,7 +576,7 @@ func (deploy *NewDeploy) idleObjectReaper() {
envs, err := deploy.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
log.Fatalf("Failed to get environment list: %v", err)
deploy.logger.Fatal("failed to get environment list", zap.Error(err))
}
envList := make(map[types.UID]struct{})
@@ -570,7 +586,7 @@ func (deploy *NewDeploy) idleObjectReaper() {
funcSvcs, err := deploy.fsCache.ListOld(deploy.idlePodReapTime)
if err != nil {
log.Printf("Error reaping idle pods: %v", err)
deploy.logger.Error("error reaping idle pods", zap.Error(err))
continue
}
@@ -582,8 +598,9 @@ func (deploy *NewDeploy) idleObjectReaper() {
// For function with the environment that no longer exists, executor
// scales down the deployment as usual and prints log to notify user.
if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
log.Printf("Environment %v for function %v no longer exists",
fsvc.Environment.Metadata.Name, fsvc.Name)
deploy.logger.Error("function environment no longer exists",
zap.String("environment", fsvc.Environment.Metadata.Name),
zap.String("function", fsvc.Name))
}
fn, err := deploy.fissionClient.Functions(fsvc.Function.Namespace).Get(fsvc.Function.Name)
@@ -593,20 +610,20 @@ func (deploy *NewDeploy) idleObjectReaper() {
if k8sErrs.IsNotFound(err) && fsvc.Executor == fscache.NEWDEPLOY {
continue
}
log.Printf("Error getting function: %v", fsvc.Function.Name)
deploy.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name))
continue
}
deployObj := getDeploymentObj(fsvc.KubernetesObjects)
if deployObj == nil {
log.Printf("Error finding deployment for function %v: %v", fsvc.Function.Name, err)
deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
continue
}
currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
if err != nil {
log.Printf("Error validating deployment for function %v: %v", fsvc.Function.Name, err)
deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
continue
}
@@ -617,9 +634,9 @@ func (deploy *NewDeploy) idleObjectReaper() {
continue
}
err = scaleDeployment(deploy.kubernetesClient, deployObj.Namespace, deployObj.Name, minScale)
err = deploy.scaleDeployment(deployObj.Namespace, deployObj.Name, minScale)
if err != nil {
log.Printf("Error scaling down deployment for function %v: %v", fsvc.Function.Name, err)
deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
}
}
}
@@ -635,9 +652,12 @@ func getDeploymentObj(kubeobjs []apiv1.ObjectReference) *apiv1.ObjectReference {
return nil
}
func scaleDeployment(client *kubernetes.Clientset, deplNS string, deplName string, replicas int32) error {
log.Printf("Scale deployment %v in namespace %v to replicas %v", deplName, deplNS, replicas)
_, err := client.ExtensionsV1beta1().Deployments(deplNS).UpdateScale(deplName, &v1beta1.Scale{
func (deploy *NewDeploy) scaleDeployment(deplNS string, deplName string, replicas int32) error {
deploy.logger.Info("scaling deployment",
zap.String("deployment", deplName),
zap.String("namespace", deplNS),
zap.Int32("replicas", replicas))
_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deplNS).UpdateScale(deplName, &v1beta1.Scale{
ObjectMeta: metav1.ObjectMeta{
Name: deplName,
Namespace: deplNS,
executor/poolmgr/funcwatcher.go
+26 -9
View File
@@ -19,7 +19,7 @@ package poolmgr
import (
"time"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -72,11 +72,15 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
// setup rolebinding is tried, if it fails, we dont return. we just log an error and move on, because :
// 1. not all functions have secrets and/or configmaps, so things will work without this rolebinding in that case.
// 2. on the contrary, when the route is tried, the env fetcher logs will show a 403 forbidden message and same will be relayed to executor.
err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.SecretConfigMapGetterRB, fn.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
if err != nil {
log.Printf("Error : %v creating %s RoleBinding", err, fission.SecretConfigMapGetterRB)
gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", fission.SecretConfigMapGetterRB))
} else {
log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in func's ns for func : %s/%s", fission.FissionFetcherSA, envNs, fn.Metadata.Namespace, fn.Metadata.Name)
gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namepsace", envNs),
zap.String("function_name", fn.Metadata.Name),
zap.String("function_namespace", fn.Metadata.Namespace))
}
if istioEnabled {
@@ -126,7 +130,11 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
// create function istio service if it does not exist
_, err = kubernetesClient.CoreV1().Services(envNs).Create(&svc)
if err != nil && !kerrors.IsAlreadyExists(err) {
log.Printf("Error creating function istio service: %v", err)
gpm.logger.Error("error creating istio service for function",
zap.Error(err),
zap.String("service_name", svcName),
zap.String("function_name", fn.Metadata.Name),
zap.Any("selectors", sel))
}
}
},
@@ -149,7 +157,11 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
// delete function istio service
err := kubernetesClient.CoreV1().Services(envNs).Delete(svcName, nil)
if err != nil && !kerrors.IsNotFound(err) {
log.Printf("Error deleting function istio service: %v", err)
gpm.logger.Error("error deleting istio service for function",
zap.Error(err),
zap.String("service_name", svcName),
zap.String("function_name", fn.Metadata.Name))
}
}
},
@@ -178,13 +190,18 @@ func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClie
envNs = newFunc.Spec.Environment.Namespace
}
err := fission.SetupRoleBinding(kubernetesClient, fission.SecretConfigMapGetterRB,
err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.SecretConfigMapGetterRB,
newFunc.Metadata.Namespace, fission.SecretConfigMapGetterCR, fission.ClusterRole,
fission.FissionFetcherSA, envNs)
if err != nil {
log.Printf("Error : %v creating GetSecretConfigMapRoleBinding", err)
gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", fission.SecretConfigMapGetterRB))
} else {
log.Printf("Set up rolebinding for fetcher SA in func's env ns : %s, in func's ns : %s, for func : %s", newFunc.Spec.Environment.Namespace, newFunc.Metadata.Namespace, newFunc.Metadata.Name)
gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namepsace", envNs),
zap.String("function_name", newFunc.Metadata.Name),
zap.String("function_namespace", newFunc.Metadata.Namespace))
}
}
},
executor/poolmgr/gp.go
+37 -30
View File
@@ -19,7 +19,6 @@ package poolmgr
import (
"context"
"fmt"
"log"
"math/rand"
"net"
"os"
@@ -30,6 +29,7 @@ import (
"github.com/dchest/uniuri"
multierror "github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
"k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -46,6 +46,7 @@ import (
type (
GenericPool struct {
logger *zap.Logger
env *crd.Environment
replicas int32 // num idle pods
deployment *v1beta1.Deployment // kubernetes deployment
@@ -83,6 +84,7 @@ type (
)
func MakeGenericPool(
logger *zap.Logger,
fissionClient *crd.FissionClient,
kubernetesClient *kubernetes.Clientset,
env *crd.Environment,
@@ -94,7 +96,9 @@ func MakeGenericPool(
enableIstio bool,
collectorEndpoint string) (*GenericPool, error) {
log.Printf("Creating pool for environment %v", env.Metadata)
gpLogger := logger.Named("generic_pool")
gpLogger.Info("creating pool", zap.Any("environment", env.Metadata))
fetcherImage := os.Getenv("FETCHER_IMAGE")
if len(fetcherImage) == 0 {
@@ -104,6 +108,7 @@ func MakeGenericPool(
// TODO: in general we need to provide the user a way to configure pools. Initial
// replicas, autoscaling params, various timeouts, etc.
gp := &GenericPool{
logger: gpLogger,
env: env,
replicas: initialReplicas, // TODO make this an env param instead?
requestChannel: make(chan *choosePodRequest),
@@ -128,13 +133,12 @@ func MakeGenericPool(
gp.runtimeImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY"))
gp.fetcherImagePullPolicy = fission.GetImagePullPolicy(os.Getenv("FETCHER_IMAGE_PULL_POLICY"))
log.Printf("fetcher image: %v, pull policy: %v", gp.fetcherImage, gp.fetcherImagePullPolicy)
gpLogger.Info("fetcher", zap.String("image", gp.fetcherImage), zap.Any("pull_policy", gp.fetcherImagePullPolicy))
// create fetcher SA in this ns, if not already created
_, err := fission.SetupSA(gp.kubernetesClient, fission.FissionFetcherSA, gp.namespace)
if err != nil {
log.Printf("Error : %v creating fetcher SA in ns : %s", err, gp.namespace)
return nil, err
return nil, errors.Wrapf(err, "error creating fetcher service account in namespace %q", gp.namespace)
}
// Labels for generic deployment/RS/pods.
@@ -145,7 +149,7 @@ func MakeGenericPool(
if err != nil {
return nil, err
}
log.Printf("[%v] Deployment created", env.Metadata)
gpLogger.Info("deployment created", zap.Any("environment", env.Metadata))
go gp.choosePodService()
@@ -195,7 +199,7 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
for {
// Retries took too long, error out.
if time.Since(startTime) > gp.podReadyTimeout {
log.Printf("[%v] Erroring out, timed out", newLabels)
gp.logger.Error("timed out waiting for pod", zap.Any("labels", newLabels), zap.Duration("timeout", gp.podReadyTimeout))
return nil, errors.New("timeout: waited too long to get a ready pod")
}
@@ -220,7 +224,10 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
// add it to the list of ready pods
readyPods = append(readyPods, &pod)
}
log.Printf("[%v] found %v ready pods of %v total", newLabels, len(readyPods), len(podList.Items))
gp.logger.Info("found ready pods",
zap.Any("labels", newLabels),
zap.Int("ready_count", len(readyPods)),
zap.Int("total", len(podList.Items)))
// If there are no ready pods, wait and retry.
if len(readyPods) == 0 {
@@ -241,14 +248,14 @@ func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, erro
// modified, this should fail; in that case just
// retry.
chosenPod.ObjectMeta.Labels = newLabels
log.Printf("relabeling pod: [%v]", chosenPod.ObjectMeta.Name)
gp.logger.Info("relabeling pod", zap.String("pod", chosenPod.ObjectMeta.Name))
_, err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Update(chosenPod)
if err != nil {
log.Printf("failed to relabel pod [%v]: %v", chosenPod.ObjectMeta.Name, err)
gp.logger.Error("failed to relabel pod", zap.Error(err), zap.String("pod", chosenPod.ObjectMeta.Name))
continue
}
}
log.Printf("Chosen pod: %v (in %v)", chosenPod.ObjectMeta.Name, time.Since(startTime))
gp.logger.Info("chose pod", zap.String("pod", chosenPod.ObjectMeta.Name), zap.Duration("elapsed_time", time.Since(startTime)))
return chosenPod, nil
}
}
@@ -267,7 +274,7 @@ func (gp *GenericPool) scheduleDeletePod(name string) {
// The sleep allows debugging or collecting logs from the pod before it's
// cleaned up. (We need a better solutions for both those things; log
// aggregation and storage will help.)
log.Printf("Error in pod '%v', scheduling cleanup", name)
gp.logger.Error("error in pod - scheduling cleanup", zap.String("pod", name))
// Ignore sleep here if istio feature is enabled, function pod
// will be deleted after 6 mins (terminationGracePeriodSeconds).
if !gp.useIstio {
@@ -319,7 +326,7 @@ func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metada
// tell fetcher to get the function.
fetcherUrl := gp.getSpecializeUrl(podIP)
log.Printf("[%v] calling fetcher to copy function with fetcher url: %v", metadata.Name, fetcherUrl)
gp.logger.Info("calling fetcher to copy function", zap.String("function", metadata.Name), zap.String("url", fetcherUrl))
fn, err := gp.fissionClient.
Functions(metadata.Namespace).
@@ -356,9 +363,9 @@ func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metada
},
}
log.Printf("[%v] specializing pod", metadata.Name)
gp.logger.Info("specializing pod", zap.String("function", metadata.Name))
err = fetcherClient.MakeClient(fetcherUrl).Specialize(ctx, &specializeReq)
err = fetcherClient.MakeClient(gp.logger, fetcherUrl).Specialize(ctx, &specializeReq)
if err != nil {
return err
}
@@ -550,7 +557,7 @@ func (gp *GenericPool) createPool() error {
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment)
if err != nil {
log.Printf("Error creating deployment for %s in kubernetes, err: %v", deployment.Name, err)
gp.logger.Error("error creating deployment in kubernetes", zap.Error(err), zap.String("deployment", deployment.Name))
return err
}
gp.deployment = depl
@@ -564,11 +571,9 @@ func (gp *GenericPool) waitForReadyPod() error {
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get(
gp.deployment.ObjectMeta.Name, metav1.GetOptions{})
if err != nil {
err = errors.Wrap(err, fmt.Sprintf(
"Error waiting for ready pod of deployment %v in namespace %v",
gp.deployment.ObjectMeta.Name, gp.namespace))
log.Print(err)
return err
e := "error waiting for ready pod for deployment"
gp.logger.Error(e, zap.String("deployment", gp.deployment.ObjectMeta.Name), zap.String("namespace", gp.namespace))
return fmt.Errorf("%s %q in namespace %q", e, gp.deployment.ObjectMeta.Name, gp.namespace)
}
gp.deployment = depl
@@ -582,7 +587,7 @@ func (gp *GenericPool) waitForReadyPod() error {
gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
})
if err != nil {
log.Printf("Error getting pod list after timeout waiting for ready pod: %v", err)
gp.logger.Error("error getting pod list after timeout waiting for ready pod", zap.Error(err))
}
// Since even single pod is not ready, choosing the first pod to inspect is a good approximation. In future this can be done better
@@ -593,9 +598,8 @@ func (gp *GenericPool) waitForReadyPod() error {
multierr = multierror.Append(multierr, errors.New(fmt.Sprintf("%v: %v", cStatus.State.Waiting.Reason, cStatus.State.Waiting.Message)))
}
}
return errors.Errorf(
"Timeout: waited too long for pod of deployment %v in namespace %v to be ready, error: %v",
gp.deployment.ObjectMeta.Name, gp.namespace, multierr)
return errors.Wrapf(multierr, "Timeout: waited too long for pod of deployment %v in namespace %v to be ready",
gp.deployment.ObjectMeta.Name, gp.namespace)
}
time.Sleep(1000 * time.Millisecond)
}
@@ -623,7 +627,7 @@ func (gp *GenericPool) createSvc(name string, labels map[string]string) (*apiv1.
}
func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
log.Printf("[%v] Choosing pod from pool", m.Name)
gp.logger.Info("choosing pod from pool", zap.String("function", m.Name))
newLabels := gp.labelsForFunction(m)
if gp.useIstio {
@@ -677,7 +681,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
gp.scheduleDeletePod(pod.ObjectMeta.Name)
return nil, err
}
log.Printf("Specialized pod: %v", pod.ObjectMeta.Name)
gp.logger.Info("specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
var svcHost string
if gp.useSvc && !gp.useIstio {
@@ -694,7 +698,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
}
if svc.ObjectMeta.Name != svcName {
gp.scheduleDeletePod(pod.ObjectMeta.Name)
return nil, errors.New(fmt.Sprintf("sanity check failed for svc %v", svc.ObjectMeta.Name))
return nil, errors.Errorf("sanity check failed for svc %v", svc.ObjectMeta.Name)
}
// the fission router isn't in the same namespace, so return a
@@ -704,7 +708,7 @@ func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*f
svc := fission.GetFunctionIstioServiceName(m.Name, m.Namespace)
svcHost = fmt.Sprintf("%v.%v:8888", svc, gp.namespace)
} else {
log.Printf("Using pod IP for specialized pod")
gp.logger.Info("using pod IP for specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP)
}
@@ -746,7 +750,10 @@ func (gp *GenericPool) destroy() error {
err := gp.kubernetesClient.ExtensionsV1beta1().
Deployments(gp.namespace).Delete(gp.deployment.ObjectMeta.Name, &delOpt)
if err != nil {
log.Printf("Error destroying deployment: %v", err)
gp.logger.Error("error destroying deployment",
zap.Error(err),
zap.String("deployment_name", gp.deployment.ObjectMeta.Name),
zap.String("deployment_namespace", gp.namespace))
return err
}
return nil
executor/poolmgr/gpm.go
+27 -18
View File
@@ -18,12 +18,12 @@ package poolmgr
import (
"context"
"log"
"os"
"strconv"
"strings"
"time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/kubernetes"
@@ -45,6 +45,8 @@ const (
type (
GenericPoolManager struct {
logger *zap.Logger
pools map[string]*GenericPool
kubernetesClient *kubernetes.Clientset
namespace string
@@ -78,18 +80,22 @@ type (
)
func MakeGenericPoolManager(
logger *zap.Logger,
fissionClient *crd.FissionClient,
kubernetesClient *kubernetes.Clientset,
functionNamespace string,
instanceId string) *GenericPoolManager {
gpmLogger := logger.Named("generic_pool_manager")
gpm := &GenericPoolManager{
logger: gpmLogger,
pools: make(map[string]*GenericPool),
kubernetesClient: kubernetesClient,
namespace: functionNamespace,
fissionClient: fissionClient,
functionEnv: cache.MakeCache(10*time.Second, 0),
fsCache: fscache.MakeFunctionServiceCache(),
fsCache: fscache.MakeFunctionServiceCache(gpmLogger),
instanceId: instanceId,
requestChannel: make(chan *request),
idlePodReapTime: 2 * time.Minute,
@@ -101,7 +107,7 @@ func MakeGenericPoolManager(
if len(os.Getenv("ENABLE_ISTIO")) > 0 {
istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
if err != nil {
log.Println("Failed to parse ENABLE_ISTIO")
gpmLogger.Info("failed to parse ENABLE_ISTIO")
}
gpm.enableIstio = istio
}
@@ -142,7 +148,7 @@ func (gpm *GenericPoolManager) service() {
ns = req.env.Metadata.Namespace
}
pool, err = MakeGenericPool(
pool, err = MakeGenericPool(gpm.logger,
gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize,
ns, gpm.namespace, gpm.fsCache, gpm.instanceId, gpm.enableIstio,
gpm.collectorEndpoint)
@@ -163,7 +169,7 @@ func (gpm *GenericPoolManager) service() {
if !ok || poolsize == 0 {
// Env no longer exists or pool size changed to zero
log.Printf("Destroying generic pool for environment %v", pool.env.Metadata)
gpm.logger.Info("eestroying generic pool", zap.Any("environment", pool.env.Metadata))
delete(gpm.pools, key)
// and delete the pool asynchronously.
@@ -195,7 +201,7 @@ func (gpm *GenericPoolManager) CleanupPools(envs []crd.Environment) {
func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
// from Func -> get Env
log.Printf("[%v] getting environment for function", metadata.Name)
gpm.logger.Info("getting environment for function", zap.String("function", metadata.Name))
env, err := gpm.getFunctionEnv(metadata)
if err != nil {
return nil, err
@@ -207,7 +213,7 @@ func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.
}
// from GenericPool -> get one function container
// (this also adds to the cache)
log.Printf("[%v] getting function service from pool", metadata.Name)
gpm.logger.Info("getting function service from pool", zap.String("function", metadata.Name))
return pool.GetFuncSvc(ctx, metadata)
}
@@ -228,7 +234,7 @@ func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Enviro
}
// Get env from metadata
log.Printf("[%v] getting env", m)
gpm.logger.Info("getting env", zap.Any("function", m))
env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
if err != nil {
return nil, err
@@ -247,11 +253,11 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
if fission.IsNetworkError(err) {
log.Printf("Encountered network error, retrying: %v", err)
gpm.logger.Error("encountered network error, retrying", zap.Error(err))
time.Sleep(5 * time.Second)
continue
}
log.Fatalf("Failed to get environment list: %v", err)
gpm.logger.Fatal("failed to get environment list", zap.Error(err))
}
// Create pools for all envs. TODO: we should make this a bit less eager, only
@@ -264,7 +270,7 @@ func (gpm *GenericPoolManager) eagerPoolCreator() {
if gpm.getEnvPoolsize(&env) > 0 {
_, err := gpm.GetPool(&envs.Items[i])
if err != nil {
log.Printf("eager-create pool failed: %v", err)
gpm.logger.Error("eager-create pool failed", zap.Error(err))
}
}
}
@@ -292,7 +298,7 @@ func (gpm *GenericPoolManager) IsValid(fsvc *fscache.FuncSvc) bool {
if obj.Kind == "pod" {
pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{})
if err == nil && strings.Contains(fsvc.Address, pod.Status.PodIP) && fission.IsReadyPod(pod) {
log.Printf("Valid pod address : %s", fsvc.Address)
gpm.logger.Info("valid pod address", zap.String("address", fsvc.Address))
return true
}
}
@@ -309,7 +315,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
log.Fatalf("Failed to get environment list: %v", err)
gpm.logger.Fatal("failed to get environment list", zap.Error(err))
}
envList := make(map[types.UID]struct{})
@@ -319,7 +325,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime)
if err != nil {
log.Printf("Error reaping idle pods: %v", err)
gpm.logger.Error("error reaping idle pods", zap.Error(err))
continue
}
@@ -331,8 +337,9 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
// For function with the environment that no longer exists, executor
// cleanups the idle pod as usual and prints log to notify user.
if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
log.Printf("Environment %v for function %v no longer exists",
fsvc.Environment.Metadata.Name, fsvc.Name)
gpm.logger.Info("function environment no longer exists",
zap.String("environment", fsvc.Environment.Metadata.Name),
zap.String("function", fsvc.Name))
}
if fsvc.Environment.Spec.AllowedFunctionsPerContainer == fission.AllowedFunctionsPerContainerInfinite {
@@ -341,7 +348,9 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime)
if err != nil {
log.Printf("Error deleting Kubernetes objects for fsvc '%v': %v", fsvc, err)
gpm.logger.Error("error deleting Kubernetes objects for function service",
zap.Error(err),
zap.Any("service", fsvc))
}
if !deleted {
@@ -349,7 +358,7 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
}
for _, kubeobj := range fsvc.KubernetesObjects {
reaper.CleanupKubeObject(gpm.kubernetesClient, &kubeobj)
reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &kubeobj)
}
}
}
executor/poolmgr/packagewatcher.go
+27 -8
View File
@@ -19,7 +19,7 @@ package poolmgr
import (
"time"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/client-go/kubernetes"
@@ -39,7 +39,9 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien
k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
pkg := obj.(*crd.Package)
log.Printf("List watch for package reported a new package addition: %s.%s", pkg.Metadata.Name, pkg.Metadata.Namespace)
gpm.logger.Info("list watch for package reported a new package addition",
zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namepsace", pkg.Metadata.Namespace))
// create or update role-binding for fetcher sa in env ns to be able to get the pkg contents from pkg namespace
envNs := fissionfnNamespace
@@ -49,13 +51,21 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien
// here, we return if we hit an error during rolebinding setup. this is because this rolebinding is mandatory for
// every function's package to be loaded into its env. without that, there's no point to move forward.
err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.PackageGetterRB, pkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, envNs)
if err != nil {
log.Printf("Error creating %s for package: %s.%s, err: %v", fission.PackageGetterRB, pkg.Metadata.Name, pkg.Metadata.Namespace, err)
gpm.logger.Error("error creating rolebinding for package",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namespace", pkg.Metadata.Namespace))
return
}
log.Printf("Successfully set up rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, pkg.Metadata.Namespace, pkg.Metadata.Name)
gpm.logger.Info("successfully set up rolebinding for fetcher service account",
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namespace", envNs),
zap.String("package_name", pkg.Metadata.Name),
zap.String("package_namespace", pkg.Metadata.Namespace))
},
UpdateFunc: func(oldObj, newObj interface{}) {
@@ -75,14 +85,23 @@ func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClien
envNs = newPkg.Spec.Environment.Namespace
}
err := fission.SetupRoleBinding(kubernetesClient, fission.PackageGetterRB,
err := fission.SetupRoleBinding(gpm.logger, kubernetesClient, fission.PackageGetterRB,
newPkg.Metadata.Namespace, fission.PackageGetterCR, fission.ClusterRole,
fission.FissionFetcherSA, envNs)
if err != nil {
log.Printf("Error : %v updating %s for package: %s.%s", err, fission.PackageGetterRB, newPkg.Metadata.Name, newPkg.Metadata.Namespace)
gpm.logger.Error("error updating rolebinding for package",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("package_name", newPkg.Metadata.Name),
zap.String("package_namespace", newPkg.Metadata.Namespace))
return
}
log.Printf("Updated rolebinding for fetcher SA: %s.%s, in packages's ns : %s, for pkg : %s", fission.FissionFetcherSA, envNs, newPkg.Metadata.Namespace, newPkg.Metadata.Name)
gpm.logger.Info("successfully updated rolebinding for fetcher service account",
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namespace", envNs),
zap.String("package_name", newPkg.Metadata.Name),
zap.String("package_namespace", newPkg.Metadata.Namespace))
}
},
})
executor/reaper/reaper.go
+85 -47
View File
@@ -17,11 +17,10 @@ limitations under the License.
package reaper
import (
"fmt"
"log"
"strings"
"time"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
@@ -36,24 +35,24 @@ var (
)
// CleanupOldExecutorObjects cleans up resources created by old executor instances
func CleanupOldExecutorObjects(kubernetesClient *kubernetes.Clientset, instanceId string) {
func CleanupOldExecutorObjects(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, instanceId string) {
go func() {
err := cleanup(kubernetesClient, instanceId)
err := cleanup(logger, kubernetesClient, instanceId)
if err != nil {
// TODO retry reaper; logged and ignored for now
log.Printf("Failed to reaper: %v", err)
logger.Error("Failed to cleanup old executor objects", zap.Error(err))
}
}()
}
func cleanup(client *kubernetes.Clientset, instanceId string) error {
func cleanup(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
err := cleanupServices(client, instanceId)
err := cleanupServices(logger, client, instanceId)
if err != nil {
return err
}
err = cleanupHpa(client, instanceId)
err = cleanupHpa(logger, client, instanceId)
if err != nil {
return err
}
@@ -61,7 +60,7 @@ func cleanup(client *kubernetes.Clientset, instanceId string) error {
// Deployments are used for idle pools and can be cleaned up
// immediately. (We should "adopt" these instead of creating
// a new pool.)
err = cleanupDeployments(client, instanceId)
err = cleanupDeployments(logger, client, instanceId)
if err != nil {
return err
}
@@ -73,7 +72,7 @@ func cleanup(client *kubernetes.Clientset, instanceId string) error {
// time.
time.Sleep(6 * time.Minute)
err = cleanupPods(client, instanceId)
err = cleanupPods(logger, client, instanceId)
if err != nil {
return err
}
@@ -82,31 +81,39 @@ func cleanup(client *kubernetes.Clientset, instanceId string) error {
}
// CleanupKubeObject deletes given kubernetes object
func CleanupKubeObject(kubeClient *kubernetes.Clientset, kubeobj *apiv1.ObjectReference) {
func CleanupKubeObject(logger *zap.Logger, kubeClient *kubernetes.Clientset, kubeobj *apiv1.ObjectReference) {
switch strings.ToLower(kubeobj.Kind) {
case "pod":
err := kubeClient.CoreV1().Pods(kubeobj.Namespace).Delete(kubeobj.Name, nil)
logErr(fmt.Sprintf("cleaning up pod %v ", kubeobj.Name), err)
if err != nil {
logger.Error("error cleaning up pod", zap.Error(err), zap.String("pod", kubeobj.Name))
}
case "service":
err := kubeClient.CoreV1().Services(kubeobj.Namespace).Delete(kubeobj.Name, nil)
logErr(fmt.Sprintf("cleaning up service %v ", kubeobj.Name), err)
if err != nil {
logger.Error("error cleaning up service", zap.Error(err), zap.String("service", kubeobj.Name))
}
case "deployment":
err := kubeClient.ExtensionsV1beta1().Deployments(kubeobj.Namespace).Delete(kubeobj.Name, &delOpt)
logErr(fmt.Sprintf("cleaning up deployment %v ", kubeobj.Name), err)
if err != nil {
logger.Error("error cleaning up deployment", zap.Error(err), zap.String("deployment", kubeobj.Name))
}
case "horizontalpodautoscaler":
err := kubeClient.AutoscalingV1().HorizontalPodAutoscalers(kubeobj.Namespace).Delete(kubeobj.Name, nil)
logErr(fmt.Sprintf("cleaning up horizontalpodautoscaler %v ", kubeobj.Name), err)
if err != nil {
logger.Error("error cleaning up horizontalpodautoscaler", zap.Error(err), zap.String("horizontalpodautoscaler", kubeobj.Name))
}
default:
log.Printf("There was an error identifying the object type: %v for obj: %v", kubeobj.Kind, kubeobj)
logger.Error("Could not identifying the object type to clean up", zap.String("type", kubeobj.Kind), zap.Any("object", kubeobj))
}
}
func cleanupDeployments(client *kubernetes.Clientset, instanceId string) error {
func cleanupDeployments(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
deploymentList, err := client.ExtensionsV1beta1().Deployments(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil {
return err
@@ -114,24 +121,34 @@ func cleanupDeployments(client *kubernetes.Clientset, instanceId string) error {
for _, dep := range deploymentList.Items {
id, ok := dep.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL]
if ok && id != instanceId {
log.Printf("Cleaning up deployment %v", dep.ObjectMeta.Name)
logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
logErr("cleaning up deployment", err)
if err != nil {
logger.Error("error cleaning up deployment",
zap.Error(err),
zap.String("deployment_name", dep.ObjectMeta.Name),
zap.String("deployment_namespace", dep.ObjectMeta.Namespace))
}
// ignore err
}
// Backward compatibility with older label name
pid, pok := dep.ObjectMeta.Labels[fission.POOLMGR_INSTANCEID_LABEL]
if pok && pid != instanceId {
log.Printf("Cleaning up deployment %v", dep.ObjectMeta.Name)
logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
logErr("cleaning up deployment", err)
if err != nil {
logger.Error("error cleaning up deployment",
zap.Error(err),
zap.String("deployment_name", dep.ObjectMeta.Name),
zap.String("deployment_namespace", dep.ObjectMeta.Namespace))
}
// ignore err
}
}
return nil
}
func cleanupPods(client *kubernetes.Clientset, instanceId string) error {
func cleanupPods(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
podList, err := client.CoreV1().Pods(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil {
return err
@@ -139,17 +156,27 @@ func cleanupPods(client *kubernetes.Clientset, instanceId string) error {
for _, pod := range podList.Items {
id, ok := pod.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL]
if ok && id != instanceId {
log.Printf("Cleaning up pod %v", pod.ObjectMeta.Name)
logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
logErr("cleaning up pod", err)
if err != nil {
logger.Error("error cleaning up pod",
zap.Error(err),
zap.String("pod_name", pod.ObjectMeta.Name),
zap.String("pod_namespace", pod.ObjectMeta.Namespace))
}
// ignore err
}
// Backward compatibility with older label name
pid, pok := pod.ObjectMeta.Labels[fission.POOLMGR_INSTANCEID_LABEL]
if pok && pid != instanceId {
log.Printf("Cleaning up pod %v", pod.ObjectMeta.Name)
logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
logErr("cleaning up pod", err)
if err != nil {
logger.Error("error cleaning up pod",
zap.Error(err),
zap.String("pod_name", pod.ObjectMeta.Name),
zap.String("pod_namespace", pod.ObjectMeta.Namespace))
}
// ignore err
}
@@ -157,7 +184,7 @@ func cleanupPods(client *kubernetes.Clientset, instanceId string) error {
return nil
}
func cleanupServices(client *kubernetes.Clientset, instanceId string) error {
func cleanupServices(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
svcList, err := client.CoreV1().Services(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil {
return err
@@ -165,16 +192,21 @@ func cleanupServices(client *kubernetes.Clientset, instanceId string) error {
for _, svc := range svcList.Items {
id, ok := svc.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL]
if ok && id != instanceId {
log.Printf("Cleaning up svc %v", svc.ObjectMeta.Name)
logger.Info("cleaning up service", zap.String("service", svc.ObjectMeta.Name))
err := client.CoreV1().Services(svc.ObjectMeta.Namespace).Delete(svc.ObjectMeta.Name, nil)
logErr("cleaning up service", err)
if err != nil {
logger.Error("error cleaning up service",
zap.Error(err),
zap.String("service_name", svc.ObjectMeta.Name),
zap.String("service_namespace", svc.ObjectMeta.Namespace))
}
// ignore err
}
}
return nil
}
func cleanupHpa(client *kubernetes.Clientset, instanceId string) error {
func cleanupHpa(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
hpaList, err := client.AutoscalingV1().HorizontalPodAutoscalers(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil {
return err
@@ -183,9 +215,15 @@ func cleanupHpa(client *kubernetes.Clientset, instanceId string) error {
for _, hpa := range hpaList.Items {
id, ok := hpa.ObjectMeta.Labels[fission.EXECUTOR_INSTANCEID_LABEL]
if ok && id != instanceId {
log.Printf("Cleaning up HPA %v", hpa.ObjectMeta.Name)
logger.Info("cleaning up HPA", zap.String("hpa", hpa.ObjectMeta.Name))
err := client.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Delete(hpa.ObjectMeta.Name, nil)
logErr("cleaning up HPA", err)
if err != nil {
logger.Error("error cleaning up HPA",
zap.Error(err),
zap.String("hpa_name", hpa.ObjectMeta.Name),
zap.String("hpa_namespace", hpa.ObjectMeta.Namespace))
}
// ignore err
}
}
@@ -193,22 +231,16 @@ func cleanupHpa(client *kubernetes.Clientset, instanceId string) error {
}
func logErr(msg string, err error) {
if err != nil {
log.Printf("Error %v: %v", msg, err)
}
}
// CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or
// deletes the rolebindings completely if there are no Service Accounts in a rolebinding object.
func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
for {
log.Println("Starting cleanupRoleBindings cycle")
logger.Info("starting cleanupRoleBindings cycle")
// get all rolebindings ( just to be efficient, one call to kubernetes )
rbList, err := client.RbacV1beta1().RoleBindings(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
if err != nil {
// something wrong, but next iteration hopefully succeeds
log.Printf("Error listing rolebindings in all ns, err: %v", err)
logger.Error("error listing role bindings in all namespaces", zap.Error(err))
continue
}
@@ -228,7 +260,7 @@ func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.Fissio
// we can list the functions once per role-binding.
funcList, err := fissionClient.Functions(roleBinding.Namespace).List(meta_v1.ListOptions{})
if err != nil {
log.Printf("Error fetching environment list in : %s", roleBinding.Namespace)
logger.Error("error fetching environment list in namespace", zap.Error(err), zap.String("namespace", roleBinding.Namespace))
continue
}
@@ -275,7 +307,7 @@ func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.Fissio
// check if there is an env obj in saNs
envList, err := fissionClient.Environments(saNs).List(meta_v1.ListOptions{})
if err != nil {
log.Printf("Error fetching environment list in : %s", saNs)
logger.Error("error fetching environment list in service account namespace", zap.Error(err), zap.String("namespace", saNs))
continue
}
@@ -313,15 +345,21 @@ func CleanupRoleBindings(client *kubernetes.Clientset, fissionClient *crd.Fissio
// finally, make a call to RemoveSAFromRoleBindingWithRetries for all the service accounts that need to be removed
// for the role-binding in this iteration
if len(saToRemove) != 0 {
log.Printf("saToRemove : %v for rolebinding : %s.%s", saToRemove, roleBinding.Name, roleBinding.Namespace)
logger.Info("removing service accounts from role binding",
zap.Any("service_accounts", saToRemove),
zap.String("role_binding_name", roleBinding.Name),
zap.String("role_binding_namespace", roleBinding.Namespace))
// call this once in the end for each role-binding
err = fission.RemoveSAFromRoleBindingWithRetries(client, roleBinding.Name, roleBinding.Namespace, saToRemove)
err = fission.RemoveSAFromRoleBindingWithRetries(logger, client, roleBinding.Name, roleBinding.Namespace, saToRemove)
if err != nil {
// if there's an error, we just log it and proceed with the next role-binding, hoping that this role-binding
// will be processed in next iteration.
log.Printf("Error removing SA : %v from rolebinding : %s.%s, err: %v", saToRemove, roleBinding.Name,
roleBinding.Namespace, err)
logger.Info("error removing service account from role binding",
zap.Error(err),
zap.Any("service_accounts", saToRemove),
zap.String("role_binding_name", roleBinding.Name),
zap.String("role_binding_namespace", roleBinding.Namespace))
}
}
}
fission-bundle/main.go
+50 -44
View File
@@ -6,10 +6,10 @@ import (
"os"
"strconv"
docopt "github.com/docopt/docopt-go"
"go.opencensus.io/exporter/jaeger"
"go.opencensus.io/trace"
docopt "github.com/docopt/docopt-go"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/buildermgr"
@@ -22,66 +22,66 @@ import (
"github.com/fission/fission/timer"
)
func runController(port int) {
controller.Start(port, false)
log.Fatalf("Error: Controller exited.")
func runController(logger *zap.Logger, port int) {
controller.Start(logger, port, false)
logger.Fatal("controller exited")
}
func runRouter(port int, executorUrl string) {
router.Start(port, executorUrl)
log.Fatalf("Error: Router exited.")
func runRouter(logger *zap.Logger, port int, executorUrl string) {
router.Start(logger, port, executorUrl)
logger.Fatal("router exited")
}
func runExecutor(port int, fissionNamespace, functionNamespace, envBuilderNamespace string) {
err := executor.StartExecutor(fissionNamespace, functionNamespace, envBuilderNamespace, port)
func runExecutor(logger *zap.Logger, port int, fissionNamespace, functionNamespace, envBuilderNamespace string) {
err := executor.StartExecutor(logger, fissionNamespace, functionNamespace, envBuilderNamespace, port)
if err != nil {
log.Fatalf("Error starting executor: %v", err)
logger.Fatal("error starting executor", zap.Error(err))
}
}
func runKubeWatcher(routerUrl string) {
err := kubewatcher.Start(routerUrl)
func runKubeWatcher(logger *zap.Logger, routerUrl string) {
err := kubewatcher.Start(logger, routerUrl)
if err != nil {
log.Fatalf("Error starting kubewatcher: %v", err)
logger.Fatal("error starting kubewatcher", zap.Error(err))
}
}
func runTimer(routerUrl string) {
err := timer.Start(routerUrl)
func runTimer(logger *zap.Logger, routerUrl string) {
err := timer.Start(logger, routerUrl)
if err != nil {
log.Fatalf("Error starting timer: %v", err)
logger.Fatal("error starting timer", zap.Error(err))
}
}
func runMessageQueueMgr(routerUrl string) {
err := messagequeue.Start(routerUrl)
func runMessageQueueMgr(logger *zap.Logger, routerUrl string) {
err := messagequeue.Start(logger, routerUrl)
if err != nil {
log.Fatalf("Error starting timer: %v", err)
logger.Fatal("error starting message queue manager", zap.Error(err))
}
}
func runStorageSvc(port int, filePath string) {
func runStorageSvc(logger *zap.Logger, port int, filePath string) {
subdir := os.Getenv("SUBDIR")
if len(subdir) == 0 {
subdir = "fission-functions"
}
enableArchivePruner := true
storagesvc.RunStorageService(storagesvc.StorageTypeLocal,
storagesvc.RunStorageService(logger, storagesvc.StorageTypeLocal,
filePath, subdir, port, enableArchivePruner)
}
func runBuilderMgr(storageSvcUrl string, envBuilderNamespace string) {
err := buildermgr.Start(storageSvcUrl, envBuilderNamespace)
func runBuilderMgr(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) {
err := buildermgr.Start(logger, storageSvcUrl, envBuilderNamespace)
if err != nil {
log.Fatalf("Error starting buildermgr: %v", err)
logger.Fatal("error starting builder manager", zap.Error(err))
}
}
func getPort(portArg interface{}) int {
func getPort(logger *zap.Logger, portArg interface{}) int {
portArgStr := portArg.(string)
port, err := strconv.Atoi(portArgStr)
if err != nil {
log.Fatalf("Error: invalid port number '%v'", portArgStr)
logger.Fatal("invalid port number", zap.Error(err), zap.String("port", portArgStr))
}
return port
}
@@ -94,10 +94,10 @@ func getStringArgWithDefault(arg interface{}, defaultValue string) string {
}
}
func registerTraceExporter(arguments map[string]interface{}) error {
func registerTraceExporter(logger *zap.Logger, arguments map[string]interface{}) error {
collectorEndpoint := getStringArgWithDefault(arguments["--collectorEndpoint"], "")
if collectorEndpoint == "" {
log.Print("Skipping trace exporter registration")
logger.Info("skipping trace exporter registration")
return nil
}
@@ -193,15 +193,21 @@ Options:
--builderMgr Start builder manager.
--version Print version information
`
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
version := fmt.Sprintf("Fission Bundle Version: %v", fission.BuildInfo().String())
arguments, err := docopt.Parse(usage, nil, true, version, false)
if err != nil {
log.Fatalf("Error: %v", err)
logger.Fatal("Could not parse command line arguments", zap.Error(err))
}
err = registerTraceExporter(arguments)
err = registerTraceExporter(logger, arguments)
if err != nil {
log.Fatalf("Error: %v", err)
logger.Fatal("Could not register trace exporter", zap.Error(err), zap.Any("argument", arguments))
}
functionNs := getStringArgWithDefault(arguments["--namespace"], "fission-function")
@@ -213,40 +219,40 @@ Options:
storageSvcUrl := getStringArgWithDefault(arguments["--storageSvcUrl"], "http://storagesvc.fission")
if arguments["--controllerPort"] != nil {
port := getPort(arguments["--controllerPort"])
runController(port)
port := getPort(logger, arguments["--controllerPort"])
runController(logger, port)
}
if arguments["--routerPort"] != nil {
port := getPort(arguments["--routerPort"])
runRouter(port, executorUrl)
port := getPort(logger, arguments["--routerPort"])
runRouter(logger, port, executorUrl)
}
if arguments["--executorPort"] != nil {
port := getPort(arguments["--executorPort"])
runExecutor(port, fissionNs, functionNs, envBuilderNs)
port := getPort(logger, arguments["--executorPort"])
runExecutor(logger, port, fissionNs, functionNs, envBuilderNs)
}
if arguments["--kubewatcher"] == true {
runKubeWatcher(routerUrl)
runKubeWatcher(logger, routerUrl)
}
if arguments["--timer"] == true {
runTimer(routerUrl)
runTimer(logger, routerUrl)
}
if arguments["--mqt"] == true {
runMessageQueueMgr(routerUrl)
runMessageQueueMgr(logger, routerUrl)
}
if arguments["--builderMgr"] == true {
runBuilderMgr(storageSvcUrl, envBuilderNs)
runBuilderMgr(logger, storageSvcUrl, envBuilderNs)
}
if arguments["--storageServicePort"] != nil {
port := getPort(arguments["--storageServicePort"])
port := getPort(logger, arguments["--storageServicePort"])
filePath := arguments["--filePath"].(string)
runStorageSvc(port, filePath)
runStorageSvc(logger, port, filePath)
}
select {}
fission/logdb/influxdb.go
+1 -1
View File
@@ -19,7 +19,6 @@ package logdb
import (
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"sort"
@@ -30,6 +29,7 @@ import (
influxdbClient "github.com/influxdata/influxdb/client/v2"
"github.com/fission/fission"
"github.com/fission/fission/fission/log"
)
const (
fission/logdb/logdb.go
+2 -4
View File
@@ -17,9 +17,8 @@ limitations under the License.
package logdb
import (
"fmt"
"time"
log "github.com/sirupsen/logrus"
)
const (
@@ -55,6 +54,5 @@ func GetLogDB(dbType string, serverURL string) (LogDatabase, error) {
case INFLUXDB:
return NewInfluxDB(serverURL)
}
log.Fatalf("Log database type is incorrect, now only support %s", INFLUXDB)
return nil, nil
return nil, fmt.Errorf("log database type is incorrect, now only support %s", INFLUXDB)
}
fission/recorder.go
+1 -1
View File
@@ -18,7 +18,6 @@ package main
import (
"fmt"
"log"
"os"
"strings"
"text/tabwriter"
@@ -29,6 +28,7 @@ import (
"github.com/fission/fission"
"github.com/fission/fission/crd"
"github.com/fission/fission/fission/log"
"github.com/fission/fission/fission/util"
)
fission/records.go
+3 -3
View File
@@ -18,12 +18,12 @@ package main
import (
"fmt"
"log"
"os"
"text/tabwriter"
"github.com/urfave/cli"
"github.com/fission/fission/fission/log"
"github.com/fission/fission/fission/util"
"github.com/fission/fission/redis/build/gen"
)
@@ -31,7 +31,7 @@ import (
func recordsView(c *cli.Context) error {
var verbosity int
if c.Bool("v") && c.Bool("vv") {
log.Fatal("Conflicting verbosity levels, use either --v or --vv")
log.Fatal("conflicting verbosity levels, use either --v or --vv")
}
if c.Bool("v") {
verbosity = 1
@@ -47,7 +47,7 @@ func recordsView(c *cli.Context) error {
//Refuse multiple filters for now
if multipleFiltersSpecified(function, trigger, from+to) {
log.Fatal("Maximum of one filter is currently supported, either --function, --trigger, or --from,--to")
log.Fatal("maximum of one filter is currently supported, either --function, --trigger, or --from,--to")
}
if len(function) != 0 {
fission/replay.go
+1 -1
View File
@@ -18,10 +18,10 @@ package main
import (
"fmt"
"log"
"os"
"text/tabwriter"
"github.com/fission/fission/fission/log"
"github.com/fission/fission/fission/util"
"github.com/urfave/cli"
)
fission/support/resources/crd.go
+10 -9
View File
@@ -17,13 +17,14 @@ limitations under the License.
package resources
import (
"log"
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/controller/client"
"github.com/fission/fission/crd"
"github.com/fission/fission/fission/log"
)
const (
@@ -52,7 +53,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdEnvironment:
items, err := res.client.EnvironmentList(metav1.NamespaceAll)
if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err)
log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return
}
@@ -64,7 +65,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdFunction:
items, err := res.client.FunctionList(metav1.NamespaceAll)
if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err)
log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return
}
@@ -76,7 +77,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdPackage:
items, err := res.client.PackageList(metav1.NamespaceAll)
if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err)
log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return
}
@@ -89,7 +90,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdHttpTrigger:
items, err := res.client.HTTPTriggerList(metav1.NamespaceAll)
if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err)
log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return
}
@@ -101,7 +102,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdKubeWatcher:
items, err := res.client.WatchList(metav1.NamespaceAll)
if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err)
log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return
}
@@ -116,7 +117,7 @@ func (res CrdDumper) Dump(dumpDir string) {
for _, mqType := range []string{fission.MessageQueueTypeNats, fission.MessageQueueTypeASQ} {
l, err := res.client.MessageQueueTriggerList(mqType, metav1.NamespaceAll)
if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err)
log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
break
}
triggers = append(triggers, l...)
@@ -130,7 +131,7 @@ func (res CrdDumper) Dump(dumpDir string) {
case CrdTimeTrigger:
items, err := res.client.TimeTriggerList(metav1.NamespaceAll)
if err != nil {
log.Printf("Error getting %v list: %v", res.crdType, err)
log.Info(fmt.Sprintf("Error getting %v list: %v", res.crdType, err))
return
}
@@ -140,7 +141,7 @@ func (res CrdDumper) Dump(dumpDir string) {
}
default:
log.Printf("Unknown type: %v", res.crdType)
log.Info(fmt.Sprintf("Unknown type: %v", res.crdType))
}
}
fission/support/resources/kubernetes.go
+13 -13
View File
@@ -21,7 +21,6 @@ import (
"bytes"
"fmt"
"io"
"log"
"path/filepath"
"sync"
@@ -30,6 +29,7 @@ import (
"k8s.io/client-go/kubernetes"
"github.com/fission/fission"
"github.com/fission/fission/fission/log"
)
const (
@@ -52,7 +52,7 @@ func NewKubernetesVersion(clientset *kubernetes.Clientset) Resource {
func (res KubernetesVersion) Dump(dumpDir string) {
serverVer, err := res.client.ServerVersion()
if err != nil {
log.Printf("Error setting up kubernetes client: %v", err)
log.Info(fmt.Sprintf("Error setting up kubernetes client: %v", err))
return
}
@@ -80,7 +80,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesService:
objs, err := res.client.CoreV1().Services(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)
log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return
}
@@ -93,7 +93,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesDeployment:
objs, err := res.client.AppsV1().Deployments(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)
log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return
}
@@ -105,7 +105,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesPod:
objs, err := res.client.CoreV1().Pods(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)
log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return
}
@@ -117,7 +117,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesHPA:
objs, err := res.client.AutoscalingV2beta1().HorizontalPodAutoscalers(metav1.NamespaceAll).List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)
log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return
}
@@ -129,7 +129,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
case KubernetesNode:
objs, err := res.client.CoreV1().Nodes().List(metav1.ListOptions{LabelSelector: res.selector})
if err != nil {
log.Printf("Error getting %v list with selector %v: %v", res.objType, res.selector, err)
log.Info(fmt.Sprintf("Error getting %v list with selector %v: %v", res.objType, res.selector, err))
return
}
@@ -142,7 +142,7 @@ func (res KubernetesObjectDumper) Dump(dumpDir string) {
}
default:
log.Printf("Unknown type: %v", res.objType)
log.Info(fmt.Sprintf("Unknown type: %v", res.objType))
}
}
@@ -187,7 +187,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
Pods(metav1.NamespaceAll).
List(metav1.ListOptions{LabelSelector: res.labelSelector})
if err != nil {
log.Printf("Error getting controller list: %v", err)
log.Info(fmt.Sprintf("Error getting controller list: %v", err))
return
}
@@ -200,7 +200,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
defer wg.Done()
if !fission.IsReadyPod(&pod) {
fmt.Printf("Pod %v is not in ready state, ignore it\n", pod.Name)
log.Info(fmt.Sprintf("Pod %v is not in ready state, ignore it\n", pod.Name))
return
}
@@ -211,7 +211,7 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
stream, err := req.Stream()
if err != nil {
log.Printf("Error streaming logs for pod %v: %v", pod.Name, err)
log.Info(fmt.Sprintf("Error streaming logs for pod %v: %v", pod.Name, err))
return
}
@@ -225,13 +225,13 @@ func (res KubernetesPodLogDumper) Dump(dumpDir string) {
stream.Close()
break
}
log.Printf("Error reading logs from buffer: %v", err)
log.Info(fmt.Sprintf("Error reading logs from buffer: %v", err))
return
}
_, err = buffer.WriteString(string(line) + "\n")
if err != nil {
log.Printf("Error writing bytes to buffer: %v", err)
log.Info(fmt.Sprintf("Error writing bytes to buffer: %v", err))
}
}
fission/support/resources/resource.go
+3 -3
View File
@@ -19,13 +19,13 @@ package resources
import (
"fmt"
"io/ioutil"
"log"
"path/filepath"
"github.com/ghodss/yaml"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/fission/log"
)
type Resource interface {
@@ -40,7 +40,7 @@ func getFileName(dumpdir string, meta metav1.ObjectMeta) string {
func writeToFile(file string, obj interface{}) {
bs, err := yaml.Marshal(obj)
if err != nil {
log.Printf("Error encoding object: %v", err)
log.Info(fmt.Sprintf("Error encoding object: %v", err))
return
}
@@ -52,6 +52,6 @@ func writeToFile(file string, obj interface{}) {
err = ioutil.WriteFile(file, bs, 0644)
if err != nil {
log.Printf("Error writing file %v: %v", file, err)
log.Info(fmt.Sprintf("Error writing file %v: %v", file, err))
}
}
fission/util/version.go
+2 -1
View File
@@ -3,11 +3,12 @@ package util
import (
"fmt"
yaml "gopkg.in/yaml.v2"
"github.com/fission/fission"
"github.com/fission/fission/controller/client"
"github.com/fission/fission/fission/log"
"github.com/fission/fission/fission/plugin"
yaml "gopkg.in/yaml.v2"
)
// Versions is a container of versions of the client (and its plugins) and server (and its plugins).
glide.lock
Generated
+21 -11
View File
@@ -1,5 +1,5 @@
hash: fbb8e5b2fc64114cf4cd7158a1498f36c580e01b327c14cd35682f999ce13db3
updated: 2019-01-24T19:14:41.16237463+05:30
hash: 9c39373a78b1e542a42169b1af8fa644f62690e73a988a01bd5ad45a3639d872
updated: 2019-03-04T16:38:40.987945-07:00
imports:
- name: cloud.google.com/go
version: 3b1ae45394a234c385be014e9a488f2bb6eef821
@@ -110,12 +110,10 @@ imports:
- openstack/identity/v3/tokens
- openstack/utils
- pagination
- name: github.com/gorilla/context
version: 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
- name: github.com/gorilla/handlers
version: 7e0847f9db758cdebd26c149d0ae9d5d0b9c98ce
- name: github.com/gorilla/mux
version: e3702bed27f0d39777b0b37b664b6280e8ef8fbf
version: a7962380ca08b5a188038c69871b8d3fbdf31e89
- name: github.com/graymeta/stow
version: 7b5498c561bbfeb17e413eb96c9edaf88af3855f
subpackages:
@@ -162,7 +160,7 @@ imports:
subpackages:
- pb
- name: github.com/nats-io/nats-streaming-server
version: 7b758bb93407505bc1d0cf26d091fc281d99ca0c
version: 247d7fa2c948ab918d7b4a9f761010082e8b92ea
subpackages:
- spb
- util
@@ -203,7 +201,7 @@ imports:
- name: github.com/satori/go.uuid
version: f58768cc1a7a7e77a3bd49e98cdd21419399b6a3
- name: github.com/Shopify/sarama
version: 03a43f93cd29dc549e6d9b11892795c206f9c38c
version: 4602b5a8c6e826f9e0737865818dd43b2339a092
- name: github.com/sirupsen/logrus
version: 68cec9f21fbf3ea8d8f98c044bc6ce05f17b267a
- name: github.com/spf13/pflag
@@ -223,7 +221,7 @@ imports:
- name: github.com/urfave/cli
version: cfb38830724cc34fedffe9a2a29fb54fa9169cd1
- name: go.opencensus.io
version: 2b5032d79456124f42db6b7eb19ac6c155449dc2
version: f305e5c4e2cf345eba88de13d10de1126fa45a61
subpackages:
- exemplar
- exporter/jaeger
@@ -240,6 +238,18 @@ imports:
- trace/internal
- trace/propagation
- trace/tracestate
- name: go.uber.org/atomic
version: 1ea20fb1cbb1cc08cbd0d913a96dead89aa18289
- name: go.uber.org/multierr
version: 3c4937480c32f4c13a875a1829af76c98ca3d40a
- name: go.uber.org/zap
version: ff33455a0e382e8a81d14dd7c922020b6b5e7982
subpackages:
- buffer
- internal/bufferpool
- internal/color
- internal/exit
- zapcore
- name: golang.org/x/crypto
version: 81e90905daefcd6fd217b62423c0908922eadb30
subpackages:
@@ -307,7 +317,7 @@ imports:
- name: gopkg.in/yaml.v2
version: 670d4cfef0544295bc27a114dbac37980d83185a
- name: k8s.io/api
version: 0f11257a8a25954878633ebdc9841c67d8f83bdb
version: c89978d5f86d7427bef2fc7752732c8c60b1d188
subpackages:
- admissionregistration/v1alpha1
- admissionregistration/v1beta1
@@ -338,7 +348,7 @@ imports:
- storage/v1alpha1
- storage/v1beta1
- name: k8s.io/apiextensions-apiserver
version: f584b16eb23bd2a3fd292a027d698d95db427c5d
version: f0729a5940c5414a9b9da2f18ba8d5c4b559ec9a
subpackages:
- pkg/apis/apiextensions
- pkg/apis/apiextensions/v1beta1
@@ -346,7 +356,7 @@ imports:
- pkg/client/clientset/clientset/scheme
- pkg/client/clientset/clientset/typed/apiextensions/v1beta1
- name: k8s.io/apimachinery
version: e386b2658ed20923da8cc9250e552f082899a1ee
version: d49e237a2683fa6dc43a86c7b1b766e0219fb6e7
subpackages:
- pkg/api/errors
- pkg/api/meta
glide.yaml
+11 -3
View File
@@ -20,7 +20,9 @@ import:
version: ^1.1.0
- package: github.com/urfave/cli
version: ^1.18.1
- package: golang.org/x/tools/imports
- package: golang.org/x/tools
subpackages:
- imports
- package: golang.org/x/net
subpackages:
- context
@@ -68,12 +70,18 @@ import:
- package: github.com/prometheus/client_golang
version: v0.8.0
- package: github.com/dustin/go-humanize
- package: github.com/golang/protobuf/proto
- package: github.com/golang/protobuf
version: v1.1.0
subpackages:
- proto
- package: github.com/bsm/sarama-cluster
version: ^2.1.11
- package: github.com/Shopify/sarama
version: ^1.15.0
- package: go.opencensus.io
version: ^0.19.0
- package: google.golang.org/api/support/bundler
- package: google.golang.org/api
subpackages:
- support/bundler
- package: go.uber.org/zap
version: v1.9.1
kubewatcher/kubewatcher.go
+27 -21
View File
@@ -21,12 +21,12 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"reflect"
"strings"
"sync/atomic"
"time"
"go.uber.org/zap"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -48,6 +48,7 @@ const (
type (
KubeWatcher struct {
logger *zap.Logger
watches map[types.UID]watchSubscription
kubernetesClient *kubernetes.Clientset
requestChannel chan *kubeWatcherRequest
@@ -56,6 +57,7 @@ type (
}
watchSubscription struct {
logger *zap.Logger
watch crd.KubernetesWatchTrigger
kubeWatch watch.Interface
lastResourceVersion string
@@ -74,8 +76,9 @@ type (
}
)
func MakeKubeWatcher(kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher {
func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher {
kw := &KubeWatcher{
logger: logger.Named("kube_watcher"),
watches: make(map[types.UID]watchSubscription),
kubernetesClient: kubernetesClient,
publisher: publisher,
@@ -167,16 +170,14 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *crd.KubernetesWa
case "JOB":
wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(listOptions)
default:
msg := fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type)
log.Println(msg)
err = errors.NewBadRequest(msg)
err = errors.NewBadRequest(fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type))
}
return wi, err
}
func (kw *KubeWatcher) addWatch(w *crd.KubernetesWatchTrigger) error {
log.Printf("Adding watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference)
ws, err := MakeWatchSubscription(w, kw.kubernetesClient, kw.publisher)
kw.logger.Info("adding watch", zap.String("name", w.Metadata.Name), zap.Any("function", w.Spec.FunctionReference))
ws, err := MakeWatchSubscription(kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher)
if err != nil {
return err
}
@@ -185,7 +186,7 @@ func (kw *KubeWatcher) addWatch(w *crd.KubernetesWatchTrigger) error {
}
func (kw *KubeWatcher) removeWatch(w *crd.KubernetesWatchTrigger) error {
log.Printf("Removing watch %v: %v", w.Metadata.Name, w.Spec.FunctionReference)
kw.logger.Info("removing watch", zap.String("name", w.Metadata.Name), zap.Any("function", w.Spec.FunctionReference))
ws, ok := kw.watches[w.Metadata.UID]
if !ok {
return fission.MakeError(fission.ErrorNotFound,
@@ -211,9 +212,10 @@ func (kw *KubeWatcher) removeWatch(w *crd.KubernetesWatchTrigger) error {
// return nil
// }
func MakeWatchSubscription(w *crd.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) {
func MakeWatchSubscription(logger *zap.Logger, w *crd.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) {
var stopped int32 = 0
ws := &watchSubscription{
logger: logger.Named("watch_subscription"),
watch: *w,
kubeWatch: nil,
stopped: &stopped,
@@ -234,8 +236,11 @@ func MakeWatchSubscription(w *crd.KubernetesWatchTrigger, kubeClient *kubernetes
func (ws *watchSubscription) restartWatch() error {
retries := 60
for {
log.Printf("(re)starting watch %v (ns:%v type:%v) at rv:%v",
ws.watch.Metadata, ws.watch.Spec.Namespace, ws.watch.Spec.Type, ws.lastResourceVersion)
ws.logger.Info("(re)starting watch",
zap.Any("watch", ws.watch.Metadata),
zap.String("namespace", ws.watch.Spec.Namespace),
zap.String("type", ws.watch.Spec.Type),
zap.String("last_resource_version", ws.lastResourceVersion))
wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion)
if err != nil {
retries--
@@ -260,7 +265,7 @@ func getResourceVersion(obj runtime.Object) (string, error) {
}
func (ws *watchSubscription) eventDispatchLoop() {
log.Println("Listening to watch ", ws.watch.Metadata.Name)
ws.logger.Info("listening to watch", zap.String("name", ws.watch.Metadata.Name))
for {
// check watchSubscription is stopped or not before waiting for event
// comes from the kubeWatch.ResultChan(). This fix the edge case that
@@ -273,14 +278,14 @@ func (ws *watchSubscription) eventDispatchLoop() {
if !more {
if ws.isStopped() {
// watch is removed by user.
log.Println("Watch stopped", ws.watch.Metadata.Name)
ws.logger.Info("watch stopped", zap.String("watch_name", ws.watch.Metadata.Name))
return
} else {
// watch closed due to timeout, restart it.
log.Printf("Watch %v timed out, restarting", ws.watch.Metadata.Name)
ws.logger.Info("watch timed out - restarting", zap.String("watch_name", ws.watch.Metadata.Name))
err := ws.restartWatch()
if err != nil {
log.Panicf("Failed to restart watch: %v", err)
ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name))
}
continue
}
@@ -288,19 +293,19 @@ func (ws *watchSubscription) eventDispatchLoop() {
if ev.Type == watch.Error {
e := errors.FromObject(ev.Object)
log.Printf("Watch error, retrying in a second: %v", e)
ws.logger.Info("watch error - retrying after one second", zap.Error(e), zap.String("watch_name", ws.watch.Metadata.Name))
// Start from the beginning to get around "too old resource version"
ws.lastResourceVersion = ""
time.Sleep(time.Second)
err := ws.restartWatch()
if err != nil {
log.Panicf("Failed to restart watch: %v", err)
ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name))
}
continue
}
rv, err := getResourceVersion(ev.Object)
if err != nil {
log.Printf("Error getting resourceVersion from object: %v", err)
ws.logger.Error("error getting resourceVersion from object", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name))
} else {
ws.lastResourceVersion = rv
}
@@ -309,7 +314,7 @@ func (ws *watchSubscription) eventDispatchLoop() {
var buf bytes.Buffer
err = printKubernetesObject(ev.Object, &buf)
if err != nil {
log.Printf("Failed to serialize object: %v", err)
ws.logger.Error("failed to serialize object", zap.Error(err), zap.String("watch_name", ws.watch.Metadata.Name))
// TODO send a POST request indicating error
}
@@ -322,8 +327,9 @@ func (ws *watchSubscription) eventDispatchLoop() {
// TODO support other function ref types. Or perhaps delegate to router?
if ws.watch.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName {
log.Printf("Error: unsupported function ref type: %v, can't publish event",
ws.watch.Spec.FunctionReference.Type)
ws.logger.Error("unsupported function ref type - cannot publish event",
zap.Any("type", ws.watch.Spec.FunctionReference.Type),
zap.String("watch_name", ws.watch.Metadata.Name))
continue
}
kubewatcher/main.go
+8 -7
View File
@@ -17,26 +17,27 @@ limitations under the License.
package kubewatcher
import (
"log"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/crd"
"github.com/fission/fission/publisher"
)
func Start(routerUrl string) error {
func Start(logger *zap.Logger, routerUrl string) error {
fissionClient, kubeClient, _, err := crd.MakeFissionClient()
if err != nil {
return err
return errors.Wrap(err, "failed to get fission or kubernetes client")
}
err = fissionClient.WaitForCRDs()
if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err)
return errors.Wrap(err, "error waiting for CRDs")
}
poster := publisher.MakeWebhookPublisher(routerUrl)
kubeWatch := MakeKubeWatcher(kubeClient, poster)
MakeWatchSync(fissionClient, kubeWatch)
poster := publisher.MakeWebhookPublisher(logger, routerUrl)
kubeWatch := MakeKubeWatcher(logger, kubeClient, poster)
MakeWatchSync(logger, fissionClient, kubeWatch)
return nil
}
kubewatcher/watchSync.go
+5 -3
View File
@@ -17,9 +17,9 @@ limitations under the License.
package kubewatcher
import (
"log"
"time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/crd"
@@ -27,13 +27,15 @@ import (
type (
WatchSync struct {
logger *zap.Logger
client *crd.FissionClient
kubeWatcher *KubeWatcher
}
)
func MakeWatchSync(client *crd.FissionClient, kubeWatcher *KubeWatcher) *WatchSync {
func MakeWatchSync(logger *zap.Logger, client *crd.FissionClient, kubeWatcher *KubeWatcher) *WatchSync {
ws := &WatchSync{
logger: logger.Named("watch_sync"),
client: client,
kubeWatcher: kubeWatcher,
}
@@ -46,7 +48,7 @@ func (ws *WatchSync) syncSvc() {
for {
watches, err := ws.client.KubernetesWatchTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
log.Fatalf("Failed to get Kubernetes watch trigger list: %v", err)
ws.logger.Fatal("failed to get Kubernetes watch trigger list", zap.Error(err))
}
ws.kubeWatcher.Sync(watches.Items)
mqtrigger/main.go
+7 -5
View File
@@ -17,22 +17,24 @@ limitations under the License.
package messagequeue
import (
"log"
"os"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/crd"
"github.com/fission/fission/mqtrigger/messageQueue"
)
func Start(routerUrl string) error {
func Start(logger *zap.Logger, routerUrl string) error {
fissionClient, _, _, err := crd.MakeFissionClient()
if err != nil {
log.Fatalf("Failed to get fission client: %v", err)
return errors.Wrap(err, "failed to get fission or kubernetes client")
}
err = fissionClient.WaitForCRDs()
if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err)
return errors.Wrap(err, "error waiting for CRDs")
}
// Message queue type: nats is the only supported one for now
@@ -42,6 +44,6 @@ func Start(routerUrl string) error {
MQType: mqType,
Url: mqUrl,
}
messageQueue.MakeMessageQueueTriggerManager(fissionClient, routerUrl, mqCfg)
messageQueue.MakeMessageQueueTriggerManager(logger, fissionClient, routerUrl, mqCfg)
return nil
}
mqtrigger/messageQueue/asq.go
+44 -27
View File
@@ -19,7 +19,6 @@ package messageQueue
import (
"bytes"
"encoding/base64"
"errors"
"fmt"
"io/ioutil"
"net/http"
@@ -29,12 +28,12 @@ import (
"sync"
"time"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/crd"
log "github.com/sirupsen/logrus"
"github.com/Azure/azure-sdk-for-go/storage"
)
// TODO: some of these constants should probably be environment variables
@@ -55,6 +54,7 @@ const (
// AzureStorageConnection represents an Azure storage connection.
type AzureStorageConnection struct {
logger *zap.Logger
routerURL string
service AzureQueueService
httpClient AzureHTTPClient
@@ -173,7 +173,7 @@ func newAzureQueueService(client storage.Client) AzureQueueService {
}
}
func newAzureStorageConnection(routerURL string, config MessageQueueConfig) (MessageQueue, error) {
func newAzureStorageConnection(logger *zap.Logger, routerURL string, config MessageQueueConfig) (MessageQueue, error) {
account := os.Getenv("AZURE_STORAGE_ACCOUNT_NAME")
if len(account) == 0 {
return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_NAME' is not set")
@@ -184,13 +184,14 @@ func newAzureStorageConnection(routerURL string, config MessageQueueConfig) (Mes
return nil, errors.New("Required environment variable 'AZURE_STORAGE_ACCOUNT_KEY' is not set")
}
log.Infof("Creating Azure storage connection to storage account '%s'.", account)
logger.Info("creating Azure storage connection to storage account", zap.String("account", account))
client, err := storage.NewBasicClient(account, key)
if err != nil {
return nil, fmt.Errorf("Failed to Azure create storage client: %v", err)
return nil, errors.Wrap(err, "failed to create Azure storage client")
}
return &AzureStorageConnection{
logger: logger.Named("azue_storage"),
routerURL: routerURL,
service: newAzureQueueService(client),
httpClient: &http.Client{
@@ -200,10 +201,10 @@ func newAzureStorageConnection(routerURL string, config MessageQueueConfig) (Mes
}
func (asc AzureStorageConnection) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscription, error) {
log.Infof("Subscribing to Azure storage queue '%s'.", trigger.Spec.Topic)
asc.logger.Info("subscribing to Azure storage queue", zap.String("queue", trigger.Spec.Topic))
if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName {
return nil, fmt.Errorf("Unsupported function reference type (%v) for trigger %v", trigger.Spec.FunctionReference.Type, trigger.Metadata.Name)
return nil, fmt.Errorf("unsupported function reference type (%v) for trigger %q", trigger.Spec.FunctionReference.Type, trigger.Metadata.Name)
}
subscription := &AzureQueueSubscription{
@@ -226,7 +227,7 @@ func (asc AzureStorageConnection) subscribe(trigger *crd.MessageQueueTrigger) (m
func (asc AzureStorageConnection) unsubscribe(subscription messageQueueSubscription) error {
sub := subscription.(*AzureQueueSubscription)
log.Infof("Unsubscribing from Azure storage queue '%s'.", sub.queueName)
asc.logger.Info("unsubscribing from Azure storage queue", zap.String("queue", sub.queueName))
// Let the worker know we've unsubscribed
sub.unsubscribe <- true
@@ -245,7 +246,7 @@ func runAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubsc
timer := time.NewTimer(AzureQueuePollingInterval)
for {
log.Infof("Waiting for %v before polling Azure storage queue '%s'.", AzureQueuePollingInterval, sub.queueName)
conn.logger.Info("waiting before polling Azure storage queue", zap.Duration("interval_length", AzureQueuePollingInterval), zap.String("queue", sub.queueName))
select {
case <-sub.unsubscribe:
timer.Stop()
@@ -261,18 +262,18 @@ func runAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubsc
}
func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubscription, wg *sync.WaitGroup) {
log.Infof("Polling messages for Azure storage queue '%s'.", sub.queueName)
conn.logger.Info("polling for messages from Azure storage queue", zap.String("queue", sub.queueName))
err := sub.queue.Create(nil)
if err != nil {
log.Errorf("Failed to create message queue '%s': %v", sub.queueName, err)
conn.logger.Error("failed to create message queue", zap.Error(err), zap.String("queue", sub.queueName))
return
}
for {
err := sub.queue.Create(nil)
if err != nil {
log.Errorf("Failed to create message queue '%s': %v", sub.queueName, err)
conn.logger.Error("failed to create message queue", zap.Error(err), zap.String("queue", sub.queueName))
return
}
@@ -281,7 +282,7 @@ func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubs
VisibilityTimeout: int(AzureMessageVisibilityTimeout / time.Second),
})
if err != nil {
log.Errorf("Failed to retrieve messages from Azure storage queue '%s': %v", sub.queueName, err)
conn.logger.Error("failed to retrieve messages from Azure storage queue", zap.Error(err), zap.String("queue", sub.queueName))
break
}
if len(messages) == 0 {
@@ -301,15 +302,15 @@ func pollAzureQueueSubscription(conn AzureStorageConnection, sub *AzureQueueSubs
func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscription, message AzureMessage) {
defer message.Delete(nil)
log.Printf("Making HTTP request to %s.", sub.functionURL)
conn.logger.Info("making HTTP request to invoke function", zap.String("function_url", sub.functionURL))
for i := 0; i <= AzureQueueRetryLimit; i++ {
if i > 0 {
log.Infof("Retry #%d for request to %s.", i, sub.functionURL)
conn.logger.Info("retrying function invocation", zap.Int("retry", i), zap.String("function_url", sub.functionURL))
}
request, err := http.NewRequest("POST", sub.functionURL, bytes.NewReader(message.Bytes()))
if err != nil {
log.Errorf("Failed to create HTTP request to %s: %v", sub.functionURL, err)
conn.logger.Error("failed to create HTTP request to invoke function", zap.Error(err), zap.String("function_url", sub.functionURL))
continue
}
@@ -324,19 +325,22 @@ func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscri
response, err := conn.httpClient.Do(request)
if err != nil {
log.Errorf("Request to %s failed: %v", sub.functionURL, err)
conn.logger.Error("sending function invocation request failed", zap.Error(err), zap.String("function_url", sub.functionURL))
continue
}
defer response.Body.Close()
body, err := ioutil.ReadAll(response.Body)
if err != nil {
log.Errorf("Failed to read response body from %s: %v.", sub.functionURL, err)
conn.logger.Error("failed to read response body from function invocation", zap.Error(err), zap.String("function_url", sub.functionURL))
continue
}
if response.StatusCode < 200 || response.StatusCode >= 300 {
log.Printf("Request to %s returned failure: %s (%d).", sub.functionURL, string(body), response.StatusCode)
conn.logger.Error("function invocation request returned a failure status code",
zap.String("function_url", sub.functionURL),
zap.String("body", string(body)),
zap.Int("status_code", response.StatusCode))
continue
}
@@ -344,14 +348,19 @@ func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscri
outputQueue := conn.service.GetQueue(sub.outputQueueName)
err = outputQueue.Create(nil)
if err != nil {
log.Errorf("Failed to create output queue '%s': %v.", sub.outputQueueName, err)
conn.logger.Error("failed to create output queue",
zap.Error(err),
zap.String("output_queue", sub.outputQueueName),
zap.String("function_url", sub.functionURL))
return
}
outputMessage := outputQueue.NewMessage(string(body))
err = outputMessage.Put(nil)
if err != nil {
log.Errorf("Failed to post response body from %s to output queue '%s': %v.", sub.functionURL, sub.outputQueueName, err)
conn.logger.Error("failed to post response body from function invocation to output queue",
zap.String("output_queue", sub.outputQueueName),
zap.String("function_url", sub.functionURL))
return
}
}
@@ -360,20 +369,28 @@ func invokeTriggeredFunction(conn AzureStorageConnection, sub *AzureQueueSubscri
return
}
log.Errorf("Request to %s failed after %d retries; moving message to poison queue.", sub.functionURL, AzureQueueRetryLimit)
conn.logger.Error("function invocation retired too many times - moving message to poison queue",
zap.Int("retry_limit", AzureQueueRetryLimit),
zap.String("function_url", sub.functionURL))
poisonQueueName := sub.queueName + AzurePoisonQueueSuffix
poisonQueue := conn.service.GetQueue(poisonQueueName)
err := poisonQueue.Create(nil)
if err != nil {
log.Errorf("Failed to create poison queue '%s': %v", poisonQueueName, err)
conn.logger.Error("failed to create poison queue",
zap.Error(err),
zap.String("poison_queue_name", poisonQueueName),
zap.String("function_url", sub.functionURL))
return
}
poisonMessage := poisonQueue.NewMessage(string(message.Bytes()))
err = poisonMessage.Put(nil)
if err != nil {
log.Errorf("Failed to post response body from %s to output queue '%s': %v", sub.functionURL, poisonQueueName, err)
conn.logger.Error("failed to post response body from function invocation failure poison queue",
zap.Error(err),
zap.String("poison_queue_name", poisonQueueName),
zap.String("function_url", sub.functionURL))
return
}
}
mqtrigger/messageQueue/asq_test.go
+32 -8
View File
@@ -18,26 +18,33 @@ package messageQueue
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"testing"
"time"
"github.com/fission/fission"
"github.com/fission/fission/crd"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/Azure/azure-sdk-for-go/storage"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/crd"
)
const (
DummyRouterURL = "http://localhost"
)
func panicIf(err error) {
if err != nil {
log.Panicf("Error: %v", err)
}
}
type azureQueueServiceMock struct {
mock.Mock
}
@@ -103,7 +110,10 @@ func (m *azureHTTPClientMock) Do(req *http.Request) (*http.Response, error) {
}
func TestNewStorageConnectionMissingAccountName(t *testing.T) {
connection, err := newAzureStorageConnection(DummyRouterURL, MessageQueueConfig{
logger, err := zap.NewDevelopment()
panicIf(err)
connection, err := newAzureStorageConnection(logger, DummyRouterURL, MessageQueueConfig{
MQType: fission.MessageQueueTypeASQ,
Url: "",
})
@@ -112,8 +122,11 @@ func TestNewStorageConnectionMissingAccountName(t *testing.T) {
}
func TestNewStorageConnectionMissingAccessKey(t *testing.T) {
logger, err := zap.NewDevelopment()
panicIf(err)
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname")
connection, err := newAzureStorageConnection(DummyRouterURL, MessageQueueConfig{
connection, err := newAzureStorageConnection(logger, DummyRouterURL, MessageQueueConfig{
MQType: fission.MessageQueueTypeASQ,
Url: "",
})
@@ -123,9 +136,12 @@ func TestNewStorageConnectionMissingAccessKey(t *testing.T) {
}
func TestNewStorageConnection(t *testing.T) {
logger, err := zap.NewDevelopment()
panicIf(err)
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_NAME", "accountname")
_ = os.Setenv("AZURE_STORAGE_ACCOUNT_KEY", "bm90IGEga2V5")
connection, err := newAzureStorageConnection(DummyRouterURL, MessageQueueConfig{
connection, err := newAzureStorageConnection(logger, DummyRouterURL, MessageQueueConfig{
MQType: "azure-storage-queue",
Url: "",
})
@@ -276,8 +292,12 @@ func TestAzureStorageQueuePoisonMessage(t *testing.T) {
service.On("GetQueue", QueueName).Return(queue).Once()
service.On("GetQueue", QueueName+AzurePoisonQueueSuffix).Return(poisonQueue).Once()
logger, err := zap.NewDevelopment()
panicIf(err)
// Create the storage connection and subscribe to the trigger
connection := AzureStorageConnection{
logger: logger,
routerURL: DummyRouterURL,
service: service,
httpClient: httpClient,
@@ -420,8 +440,12 @@ func runAzureStorageQueueTest(t *testing.T, count int, output bool) {
service.On("GetQueue", OutputQueueName).Return(outputQueue).Times(count)
}
logger, err := zap.NewDevelopment()
panicIf(err)
// Create the storage connection and subscribe to the trigger
connection := AzureStorageConnection{
logger: logger,
routerURL: DummyRouterURL,
service: service,
httpClient: httpClient,
mqtrigger/messageQueue/kafka.go
+51 -27
View File
@@ -26,7 +26,7 @@ import (
sarama "github.com/Shopify/sarama"
cluster "github.com/bsm/sarama-cluster"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/crd"
@@ -34,30 +34,35 @@ import (
type (
Kafka struct {
logger *zap.Logger
routerUrl string
brokers []string
version sarama.KafkaVersion
}
)
func makeKafkaMessageQueue(routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) {
func makeKafkaMessageQueue(logger *zap.Logger, routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) {
if len(routerUrl) == 0 || len(mqCfg.Url) == 0 {
return nil, errors.New("The router URL or MQ URL is empty")
return nil, errors.New("the router URL or MQ URL is empty")
}
mqKafkaVersion := os.Getenv("MESSAGE_QUEUE_KAFKA_VERSION")
// Parse version string
kafkaVersion, err := sarama.ParseKafkaVersion(mqKafkaVersion)
if err != nil {
log.Warningf("Error parsing version string %q: %v. Falling back to %q", mqKafkaVersion, err, kafkaVersion)
logger.Warn("error parsing kafka version string - falling back to default",
zap.Error(err),
zap.String("failed_version", mqKafkaVersion),
zap.Any("default_version", kafkaVersion))
}
kafka := Kafka{
logger: logger.Named("kafka"),
routerUrl: routerUrl,
brokers: strings.Split(mqCfg.Url, ","),
version: kafkaVersion,
}
log.Infof("Created Queue %v", kafka)
logger.Info("created kafka queue", zap.Any("kafka", kafka))
return kafka, nil
}
@@ -66,8 +71,8 @@ func isTopicValidForKafka(topic string) bool {
}
func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscription, error) {
log.Infof("Inside kakfa subscribe %q", trigger)
log.Infof("brokers set to %q", kafka.brokers)
kafka.logger.Info("inside kakfa subscribe", zap.Any("trigger", trigger))
kafka.logger.Info("brokers set", zap.Strings("brokers", kafka.brokers))
// Create new consumer
consumerConfig := cluster.NewConfig()
@@ -75,7 +80,7 @@ func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubs
consumerConfig.Group.Return.Notifications = true
consumerConfig.Config.Version = kafka.version
consumer, err := cluster.NewConsumer(kafka.brokers, string(trigger.Metadata.UID), []string{trigger.Spec.Topic}, consumerConfig)
log.Infof("Created a new consumer: %#v", consumer)
kafka.logger.Info("created a new consumer", zap.Any("consumer", consumer))
if err != nil {
panic(err)
}
@@ -87,7 +92,7 @@ func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubs
producerConfig.Producer.Return.Successes = true
producerConfig.Version = kafka.version
producer, err := sarama.NewSyncProducer(kafka.brokers, producerConfig)
log.Infof("Created a new producer %q", producer)
kafka.logger.Info("created a new producer", zap.Any("consumer", producer))
if err != nil {
panic(err)
}
@@ -95,21 +100,21 @@ func (kafka Kafka) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubs
// consume errors
go func() {
for err := range consumer.Errors() {
log.Printf("Error: %s\n", err.Error())
kafka.logger.Error("consumer error", zap.Error(err))
}
}()
// consume notifications
go func() {
for ntf := range consumer.Notifications() {
log.Printf("Rebalanced: %+v\n", ntf)
kafka.logger.Info("consumer notification", zap.Any("notification", ntf))
}
}()
// consume messages
go func() {
for msg := range consumer.Messages() {
log.Infof("Calling message handler with value " + string(msg.Value[:]))
kafka.logger.Info("calling message handler", zap.String("message", string(msg.Value[:])))
if kafkaMsgHandler(&kafka, producer, trigger, msg) {
consumer.MarkOffset(msg, "") // mark message as processed
}
@@ -127,12 +132,13 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
var value string = string(msg.Value[:])
// Support other function ref types
if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName {
log.Fatalf("Unsupported function reference type (%v) for trigger %v",
trigger.Spec.FunctionReference.Type, trigger.Metadata.Name)
kafka.logger.Fatal("unsupported function reference type for trigger",
zap.Any("function_reference_type", trigger.Spec.FunctionReference.Type),
zap.String("trigger", trigger.Metadata.Name))
}
url := kafka.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name, trigger.Metadata.Namespace), "/")
log.Printf("Making HTTP request to %v", url)
kafka.logger.Info("making HTTP request", zap.String("url", url))
// Generate the Headers
fissionHeaders := map[string]string{
@@ -145,7 +151,9 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
// Create request
req, err := http.NewRequest("POST", url, strings.NewReader(value))
if err != nil {
log.Warningf("Request creation failed: %v", url)
kafka.logger.Error("failed to create HTTP request to invoke function",
zap.Error(err),
zap.String("function_url", url))
return false
}
@@ -156,7 +164,8 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
req.Header.Add(string(h.Key), string(h.Value))
}
} else {
log.Warningf("Headers are not supported by Kafka version %q, needs v0.11+: no record headers to add in HTTP request", kafka.version)
kafka.logger.Warn("headers are not supported by current Kafka version, needs v0.11+: no record headers to add in HTTP request",
zap.Any("current_version", kafka.version))
}
for k, v := range fissionHeaders {
@@ -169,7 +178,10 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
// Make the request
resp, err = http.DefaultClient.Do(req)
if err != nil {
log.Errorf("Error invoking function for trigger %v: %v", trigger.Metadata.Name, err)
kafka.logger.Error("sending function invocation request failed",
zap.Error(err),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
continue
}
if resp == nil {
@@ -182,18 +194,23 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
}
if resp == nil {
log.Warning("Every retry failed; final retry gave empty response.")
kafka.logger.Warn("every function invocation retry failed; final retry gave empty response",
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
return false
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
log.Infof("Got response " + string(body))
kafka.logger.Info("got response from function invocation",
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name),
zap.String("body", string(body)))
if err != nil {
errorHandler(trigger, producer, fmt.Sprintf("Request body error: %v", string(body)))
errorHandler(kafka.logger, trigger, producer, fmt.Sprintf("request body error: %v", string(body)))
return false
}
if resp.StatusCode != 200 {
errorHandler(trigger, producer, fmt.Sprintf("Request returned failure: %v", resp.StatusCode))
errorHandler(kafka.logger, trigger, producer, fmt.Sprintf("request returned failure: %v", resp.StatusCode))
return false
}
if len(trigger.Spec.ResponseTopic) > 0 {
@@ -207,7 +224,8 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
}
}
} else {
log.Warningf("Headers are not supported by Kafka version %q, needs v0.11+: dropping the headers", kafka.version)
kafka.logger.Warn("headers are not supported by current Kafka version, needs v0.11+: no record headers to add in HTTP request",
zap.Any("current_version", kafka.version))
}
_, _, err := producer.SendMessage(&sarama.ProducerMessage{
@@ -216,24 +234,30 @@ func kafkaMsgHandler(kafka *Kafka, producer sarama.SyncProducer, trigger *crd.Me
Headers: kafkaRecordHeaders,
})
if err != nil {
log.Warningf("Failed to publish message to topic %s: %v", trigger.Spec.ResponseTopic, err)
kafka.logger.Warn("failed to publish response body from function invocation to topic",
zap.Error(err),
zap.String("topic", trigger.Spec.Topic),
zap.String("function_url", url))
return false
}
}
return true
}
func errorHandler(trigger *crd.MessageQueueTrigger, producer sarama.SyncProducer, body string) {
func errorHandler(logger *zap.Logger, trigger *crd.MessageQueueTrigger, producer sarama.SyncProducer, body string) {
if len(trigger.Spec.ErrorTopic) > 0 {
_, _, err := producer.SendMessage(&sarama.ProducerMessage{
Topic: trigger.Spec.ErrorTopic,
Value: sarama.StringEncoder(body),
})
if err != nil {
log.Warningf("Failed to publish message to error topic %s: %v", trigger.Spec.ErrorTopic, err)
logger.Warn("failed to publish message to error topic",
zap.Error(err),
zap.String("topic", trigger.Spec.Topic))
return
}
} else {
log.Printf(body)
logger.Error("message received to publish to error topic, but no error topic was set",
zap.String("message", body))
}
}
mqtrigger/messageQueue/messageQueue.go
+18 -15
View File
@@ -18,9 +18,10 @@ package messageQueue
import (
"errors"
"fmt"
"time"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
@@ -50,6 +51,7 @@ type (
}
MessageQueueTriggerManager struct {
logger *zap.Logger
reqChan chan request
mqCfg MessageQueueConfig
triggers map[string]*triggerSubscription
@@ -73,27 +75,28 @@ type (
}
)
func MakeMessageQueueTriggerManager(fissionClient *crd.FissionClient, routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager {
func MakeMessageQueueTriggerManager(logger *zap.Logger, fissionClient *crd.FissionClient, routerUrl string, mqConfig MessageQueueConfig) *MessageQueueTriggerManager {
var messageQueue MessageQueue
var err error
mqTriggerMgr := MessageQueueTriggerManager{
logger: logger.Named("message_queue_trigger_manager"),
reqChan: make(chan request),
triggers: make(map[string]*triggerSubscription),
fissionClient: fissionClient,
}
switch mqConfig.MQType {
case fission.MessageQueueTypeNats:
messageQueue, err = makeNatsMessageQueue(routerUrl, mqConfig)
messageQueue, err = makeNatsMessageQueue(logger, routerUrl, mqConfig)
case fission.MessageQueueTypeASQ:
messageQueue, err = newAzureStorageConnection(routerUrl, mqConfig)
messageQueue, err = newAzureStorageConnection(logger, routerUrl, mqConfig)
case fission.MessageQueueTypeKafka:
messageQueue, err = makeKafkaMessageQueue(routerUrl, mqConfig)
messageQueue, err = makeKafkaMessageQueue(logger, routerUrl, mqConfig)
default:
err = errors.New("No matched message queue type found")
err = fmt.Errorf("no supported message queue type found for %q", mqConfig.MQType)
}
if err != nil {
log.Fatalf("Failed to connect to remote message queue server: %v", err)
logger.Fatal("failed to connect to remote message queue server", zap.Error(err))
}
mqTriggerMgr.messageQueue = messageQueue
go mqTriggerMgr.service()
@@ -109,7 +112,7 @@ func (mqt *MessageQueueTriggerManager) service() {
var err error
k := crd.CacheKey(&req.triggerSub.trigger.Metadata)
if _, ok := mqt.triggers[k]; ok {
err = errors.New("Trigger already exists")
err = errors.New("trigger already exists")
} else {
mqt.triggers[k] = req.triggerSub
}
@@ -164,11 +167,11 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
newTriggers, err := mqt.fissionClient.MessageQueueTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
if fission.IsNetworkError(err) {
log.Printf("Encounter network error, retry again: %v", err)
mqt.logger.Info("encountered network error, will retry", zap.Error(err))
time.Sleep(5 * time.Second)
continue
}
log.Fatalf("Failed to read message queue trigger list: %v", err)
mqt.logger.Fatal("failed to read message queue trigger list", zap.Error(err))
}
newTriggerMap := make(map[string]*crd.MessageQueueTrigger)
for index := range newTriggers.Items {
@@ -188,7 +191,7 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
// actually subscribe using the message queue client impl
sub, err := mqt.messageQueue.subscribe(trigger)
if err != nil {
log.Warnf("Failed to subscribe to message queue trigger %s: %v", trigger.Metadata.Name, err)
mqt.logger.Warn("failed to subscribe to message queue trigger", zap.Error(err), zap.String("trigger_name", trigger.Metadata.Name))
continue
}
@@ -200,10 +203,10 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
// add to our list
err = mqt.addTrigger(&triggerSub)
if err != nil {
log.Fatalf("Message queue trigger %s addition failed: %v", trigger.Metadata.Name, err)
mqt.logger.Fatal("adding message queue trigger failed", zap.Error(err), zap.String("trigger_name", trigger.Metadata.Name))
}
log.Infof("Message queue trigger %s created", trigger.Metadata.Name)
mqt.logger.Info("message queue trigger created", zap.String("trigger_name", trigger.Metadata.Name))
}
// remove old triggers
@@ -213,11 +216,11 @@ func (mqt *MessageQueueTriggerManager) syncTriggers() {
}
err := mqt.messageQueue.unsubscribe(triggerSub.subscription)
if err != nil {
log.Warnf("Failed to unsubscribe to trigger %s: %v", triggerSub.trigger.Metadata.Name, err)
mqt.logger.Warn("failed to unsubscribe from message queue trigger", zap.Error(err), zap.String("trigger_name", triggerSub.trigger.Metadata.Name))
continue
}
mqt.delTrigger(&triggerSub.trigger.Metadata)
log.Infof("Message queue trigger %s deleted", triggerSub.trigger.Metadata.Name)
mqt.logger.Info("message queue trigger deleted", zap.String("trigger_name", triggerSub.trigger.Metadata.Name))
}
// TODO replace with a watch
mqtrigger/messageQueue/nats.go
+36 -14
View File
@@ -18,7 +18,6 @@ package messageQueue
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"net/http"
@@ -26,7 +25,7 @@ import (
ns "github.com/nats-io/go-nats-streaming"
nsUtil "github.com/nats-io/nats-streaming-server/util"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/crd"
@@ -41,17 +40,19 @@ const (
type (
Nats struct {
logger *zap.Logger
nsConn ns.Conn
routerUrl string
}
)
func makeNatsMessageQueue(routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) {
func makeNatsMessageQueue(logger *zap.Logger, routerUrl string, mqCfg MessageQueueConfig) (MessageQueue, error) {
conn, err := ns.Connect(natsClusterID, natsClientID, ns.NatsURL(mqCfg.Url))
if err != nil {
return nil, err
}
nats := Nats{
logger: logger.Named("nats"),
nsConn: conn,
routerUrl: routerUrl,
}
@@ -62,7 +63,7 @@ func (nats Nats) subscribe(trigger *crd.MessageQueueTrigger) (messageQueueSubscr
subj := trigger.Spec.Topic
if !isTopicValidForNats(subj) {
return nil, errors.New(fmt.Sprintf("Not a valid topic: %s", trigger.Spec.Topic))
return nil, fmt.Errorf("not a valid topic: %q", trigger.Spec.Topic)
}
opts := []ns.SubscriptionOption{
@@ -96,15 +97,16 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
// Support other function ref types
if trigger.Spec.FunctionReference.Type != fission.FunctionReferenceTypeFunctionName {
log.Fatalf("Unsupported function reference type (%v) for trigger %v",
trigger.Spec.FunctionReference.Type, trigger.Metadata.Name)
nats.logger.Fatal("unsupported function reference type for trigger",
zap.Any("function_reference_type", trigger.Spec.FunctionReference.Type),
zap.String("trigger", trigger.Metadata.Name))
}
// with the addition of multi-tenancy, the users can create functions in any namespace. however,
// the triggers can only be created in the same namespace as the function.
// so essentially, function namespace = trigger namespace.
url := nats.routerUrl + "/" + strings.TrimPrefix(fission.UrlForFunction(trigger.Spec.FunctionReference.Name, trigger.Metadata.Namespace), "/")
log.Printf("Making HTTP request to %v", url)
nats.logger.Info("making HTTP request", zap.String("url", url))
headers := map[string]string{
"X-Fission-MQTrigger-Topic": trigger.Spec.Topic,
@@ -117,7 +119,9 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
req, err := http.NewRequest("POST", url, bytes.NewReader(msg.Data))
if err != nil {
log.Errorf("Could not issue POST request with message to url %v", url)
nats.logger.Error("failed to create HTTP request to invoke function",
zap.Error(err),
zap.String("function_url", url))
return
}
@@ -130,7 +134,10 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
// Make the request
resp, err = http.DefaultClient.Do(req)
if err != nil {
log.Errorf("Error invoking function for trigger %v: %v", trigger.Metadata.Name, err)
nats.logger.Error("sending function invocation request failed",
zap.Error(err),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
continue
}
if resp == nil {
@@ -143,7 +150,9 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
}
if resp == nil {
log.Warning("Every retry failed; final retry gave empty response.")
nats.logger.Warn("every function invocation retry failed; final retry gave empty response",
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
return
}
@@ -151,7 +160,10 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
body, bodyErr := ioutil.ReadAll(resp.Body)
if bodyErr != nil {
log.Warningf("Response body error: %v", bodyErr)
nats.logger.Error("error reading function invocation response",
zap.Error(err),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
return
}
@@ -160,7 +172,11 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
if len(trigger.Spec.ErrorTopic) > 0 && len(body) > 0 {
publishErr := nats.nsConn.Publish(trigger.Spec.ErrorTopic, body)
if publishErr != nil {
log.Errorf("Failed to publish error to error topic: %v", publishErr)
nats.logger.Error("failed to publish function invocation error to error topic",
zap.Error(publishErr),
zap.String("topic", trigger.Spec.ErrorTopic),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
// TODO: We will ack this message after max retries to prevent re-processing but
// this may cause message loss
}
@@ -171,13 +187,19 @@ func msgHandler(nats *Nats, trigger *crd.MessageQueueTrigger) func(*ns.Msg) {
// Trigger acks message only if a request was processed successfully
err = msg.Ack()
if err != nil {
log.Warningf("Failed to ack message: %v", err)
nats.logger.Error("failed to ack message after successful function invocation from trigger",
zap.Error(err),
zap.String("function_url", url),
zap.String("trigger", trigger.Metadata.Name))
}
if len(trigger.Spec.ResponseTopic) > 0 {
err = nats.nsConn.Publish(trigger.Spec.ResponseTopic, body)
if err != nil {
log.Warningf("Failed to publish message to topic %s: %v", trigger.Spec.ResponseTopic, err)
nats.logger.Error("failed to publish message with function invocation response to topic",
zap.Error(err),
zap.String("topic", trigger.Spec.ResponseTopic),
zap.String("trigger", trigger.Metadata.Name))
}
}
}
preupgradechecks/main.go
+13 -7
View File
@@ -17,10 +17,10 @@ limitations under the License.
package main
import (
"os"
"log"
"github.com/docopt/docopt-go"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
"github.com/fission/fission"
)
@@ -34,6 +34,12 @@ func getStringArgWithDefault(arg interface{}, defaultValue string) string {
}
func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
defer logger.Sync()
usage := `Package to perform operations needed prior to fission installation
Usage:
pre-upgrade-checks --fn-pod-namespace=<podNamespace> --envbuilder-namespace=<envBuilderNamespace>
@@ -43,20 +49,20 @@ Options:
arguments, err := docopt.Parse(usage, nil, true, fission.BuildInfo().String(), false)
if err != nil {
log.Fatalf("Error: %v", err)
logger.Fatal("Could not parse command line arguments", zap.Error(err))
}
functionPodNs := getStringArgWithDefault(arguments["--fn-pod-namespace"], "fission-function")
envBuilderNs := getStringArgWithDefault(arguments["--envbuilder-namespace"], "fission-builder")
crdBackedClient, err := makePreUpgradeTaskClient(functionPodNs, envBuilderNs)
crdBackedClient, err := makePreUpgradeTaskClient(logger, functionPodNs, envBuilderNs)
if err != nil {
log.Printf("Error creating a crd client : %v, please retry helm upgrade", err)
os.Exit(1)
logger.Fatal("error creating a crd client, please retry helm upgrade",
zap.Error(err))
}
if !crdBackedClient.IsFissionReInstall() {
log.Printf("Nothing to do since CRDs are not present on the cluster")
logger.Info("nothing to do since CRDs are not present on the cluster")
return
}
preupgradechecks/preupgradechecks.go
+39 -26
View File
@@ -18,10 +18,10 @@ package main
import (
"fmt"
"os"
multierror "github.com/hashicorp/go-multierror"
log "github.com/sirupsen/logrus"
"github.com/pkg/errors"
"go.uber.org/zap"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -33,6 +33,7 @@ import (
type (
PreUpgradeTaskClient struct {
logger *zap.Logger
fissionClient *crd.FissionClient
k8sClient *kubernetes.Clientset
apiExtClient *apiextensionsclient.Clientset
@@ -46,19 +47,14 @@ const (
FunctionCRD = "functions.fission.io"
)
func fatal(msg string) {
os.Stderr.WriteString(msg + "\n")
os.Exit(1)
}
func makePreUpgradeTaskClient(fnPodNs, envBuilderNs string) (*PreUpgradeTaskClient, error) {
func makePreUpgradeTaskClient(logger *zap.Logger, fnPodNs, envBuilderNs string) (*PreUpgradeTaskClient, error) {
fissionClient, k8sClient, apiExtClient, err := crd.MakeFissionClient()
if err != nil {
log.Errorf("Error making fission client")
return nil, err
return nil, errors.Wrap(err, "error making fission client")
}
return &PreUpgradeTaskClient{
logger: logger.Named("pre_upgrade_task_client"),
fissionClient: fissionClient,
k8sClient: k8sClient,
fnPodNs: fnPodNs,
@@ -86,7 +82,7 @@ func (client *PreUpgradeTaskClient) IsFissionReInstall() bool {
// VerifyFunctionSpecReferences verifies that a function references secrets, configmaps, pkgs in its own namespace and
// outputs a list of functions that don't adhere to this requirement.
func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() {
log.Printf("Verifying Function spec references for all functions in the cluster")
client.logger.Info("verifying function spec references for all functions in the cluster")
var result *multierror.Error
var err error
@@ -100,7 +96,9 @@ func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() {
}
if err != nil {
fatal(fmt.Sprintf("Error: %v listing functions even after %d retries", err, MaxRetries))
client.logger.Fatal("error listing functions after max retries",
zap.Error(err),
zap.Int("max_retries", MaxRetries))
}
// check that all secrets, configmaps, packages are in the same namespace
@@ -125,12 +123,12 @@ func (client *PreUpgradeTaskClient) VerifyFunctionSpecReferences() {
}
if result != nil {
log.Printf("Installation failed due to the following errors :")
log.Printf("Summary : A function cannot reference secrets, configmaps and packages outside it's own namespace")
fatal(result.Error())
client.logger.Fatal("installation failed",
zap.Error(err),
zap.String("summary", "a function cannot reference secrets, configmaps and packages outside it's own namespace"))
}
log.Printf("Function Spec References verified")
client.logger.Info("function spec references verified")
}
// deleteClusterRoleBinding deletes the clusterRoleBinding passed as an argument to it.
@@ -152,11 +150,13 @@ func (client *PreUpgradeTaskClient) RemoveClusterAdminRolesForFissionSAs() {
for _, clusterRoleBinding := range clusterRoleBindings {
err := client.deleteClusterRoleBinding(clusterRoleBinding)
if err != nil {
fatal(fmt.Sprintf("Error deleting rolebinding : %s, err : %v", clusterRoleBinding, err))
client.logger.Fatal("error deleting rolebinding",
zap.Error(err),
zap.String("role_binding", clusterRoleBinding))
}
}
log.Println("Removed cluster admin privileges for fission-builder and fission-fetcher Service Accounts")
client.logger.Info("femoved cluster admin privileges for fission-builder and fission-fetcher service accounts")
}
// NeedRoleBindings checks if there is atleast one package or function in default namespace.
@@ -181,27 +181,40 @@ func (client *PreUpgradeTaskClient) NeedRoleBindings() bool {
// Setup appropriate role bindings for fission-fetcher and fission-builder SAs
func (client *PreUpgradeTaskClient) SetupRoleBindings() {
if !client.NeedRoleBindings() {
log.Printf("No fission objects found, so no role-bindings to create")
client.logger.Info("no fission objects found, so no role-bindings to create")
return
}
// the fact that we're here implies that there had been a prior installation of fission and objects are present still
// so, we go ahead and create the role-bindings necessary for the fission-fetcher and fission-builder Service Accounts.
err := fission.SetupRoleBinding(client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs)
err := fission.SetupRoleBinding(client.logger, client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs)
if err != nil {
fatal(fmt.Sprintf("Error setting up rolebinding %s for %s.%s service account", fission.PackageGetterRB, fission.FissionFetcherSA, client.fnPodNs))
client.logger.Fatal("error setting up rolebinding for service account",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namespace", client.fnPodNs))
}
err = fission.SetupRoleBinding(client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, client.envBuilderNs)
err = fission.SetupRoleBinding(client.logger, client.k8sClient, fission.PackageGetterRB, metav1.NamespaceDefault, fission.PackageGetterCR, fission.ClusterRole, fission.FissionBuilderSA, client.envBuilderNs)
if err != nil {
fatal(fmt.Sprintf("Error setting up rolebinding %s for %s.%s service account", fission.PackageGetterRB, fission.FissionBuilderSA, client.envBuilderNs))
client.logger.Fatal("error setting up rolebinding for service account",
zap.Error(err),
zap.String("role_binding", fission.PackageGetterRB),
zap.String("service_account", fission.FissionBuilderSA),
zap.String("service_account_namespace", client.envBuilderNs))
}
err = fission.SetupRoleBinding(client.k8sClient, fission.SecretConfigMapGetterRB, metav1.NamespaceDefault, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs)
err = fission.SetupRoleBinding(client.logger, client.k8sClient, fission.SecretConfigMapGetterRB, metav1.NamespaceDefault, fission.SecretConfigMapGetterCR, fission.ClusterRole, fission.FissionFetcherSA, client.fnPodNs)
if err != nil {
fatal(fmt.Sprintf("Error setting up rolebinding %s for %s.%s service account", fission.SecretConfigMapGetterRB, fission.FissionFetcherSA, client.fnPodNs))
client.logger.Fatal("error setting up rolebinding for service account",
zap.Error(err),
zap.String("role_binding", fission.SecretConfigMapGetterRB),
zap.String("service_account", fission.FissionFetcherSA),
zap.String("service_account_namespace", client.fnPodNs))
}
log.Printf("Created role-bindings : %s and %s in default namespace", fission.PackageGetterRB, fission.SecretConfigMapGetterRB)
client.logger.Info("created rolebindings in default namespace",
zap.Strings("role_bindings", []string{fission.PackageGetterRB, fission.SecretConfigMapGetterRB}))
return
}
publisher/webhookPublisher.go
+27 -8
View File
@@ -19,15 +19,18 @@ package publisher
import (
"bytes"
"io/ioutil"
"log"
"net/http"
"strings"
"time"
"go.uber.org/zap"
)
type (
// A webhook publisher for a single URL. Satisifies the Publisher interface.
WebhookPublisher struct {
logger *zap.Logger
requestChannel chan *publishRequest
maxRetries int
@@ -44,8 +47,9 @@ type (
}
)
func MakeWebhookPublisher(baseUrl string) *WebhookPublisher {
func MakeWebhookPublisher(logger *zap.Logger, baseUrl string) *WebhookPublisher {
p := &WebhookPublisher{
logger: logger.Named("webhook_publisher"),
baseUrl: baseUrl,
requestChannel: make(chan *publishRequest, 32), // buffered channel
// TODO make this configurable
@@ -76,13 +80,16 @@ func (p *WebhookPublisher) svc() {
func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) {
url := p.baseUrl + "/" + strings.TrimPrefix(r.target, "/")
log.Printf("Making HTTP request to %v", url)
p.logger.Info("making HTTP request", zap.String("url", url))
var buf bytes.Buffer
buf.WriteString(r.body)
// Create request
req, err := http.NewRequest("POST", url, &buf)
if err != nil {
p.logger.Error("error creating request", zap.Error(err), zap.String("url", url))
}
for k, v := range r.headers {
req.Header.Set(k, v)
}
@@ -98,13 +105,25 @@ func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) {
// Log errors
if err != nil {
log.Printf("Request failed: %v", r)
p.logger.Error("request failed",
zap.Error(err),
zap.Any("request", r),
zap.String("url", url))
} else if resp.StatusCode != 200 {
log.Printf("Request returned failure: %v", resp.StatusCode)
p.logger.Error("request returned failure status code",
zap.Any("request", r),
zap.String("url", url),
zap.Int("status_code", resp.StatusCode))
body, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err == nil {
log.Printf("request error: %v", string(body))
if err != nil {
p.logger.Error("error reading error request body",
zap.Error(err),
zap.Any("request", r),
zap.String("url", url),
zap.Int("status_code", resp.StatusCode))
} else {
p.logger.Error("request error", zap.String("body", string(body)))
}
}
@@ -116,7 +135,7 @@ func (p *WebhookPublisher) makeHttpRequest(r *publishRequest) {
p.requestChannel <- r
})
} else {
log.Printf("Final retry failed, giving up on %v", url)
p.logger.Error("final retry failed, giving up", zap.String("url", url))
// Event dropped
}
}
redis/redis.go
+18 -17
View File
@@ -26,30 +26,30 @@ import (
"github.com/golang/protobuf/proto"
"github.com/gomodule/redigo/redis"
log "github.com/sirupsen/logrus"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/redis/build/gen"
)
func NewClient() redis.Conn {
func NewClient() (redis.Conn, error) {
redisIP := os.Getenv("REDIS_SERVICE_HOST") // TODO: Do this here or somewhere earlier?
redisPort := os.Getenv("REDIS_SERVICE_PORT")
redisUrl := fmt.Sprintf("%s:%s", redisIP, redisPort)
if len(redisUrl) == 0 {
log.Error("Could not reach Redis in cluster at IP ", redisUrl)
return nil
if len(redisIP) == 0 || len(redisPort) == 0 {
return nil, errors.New("redis host or port not supplied")
}
c, err := redis.Dial("tcp", redisUrl)
redisURL := fmt.Sprintf("%s:%s", redisIP, redisPort)
c, err := redis.Dial("tcp", redisURL)
if err != nil {
log.Errorf("Could not connect to Redis: %v\n", err)
return nil
return nil, errors.Wrapf(err, "could not connect to Redis at url %q", redisURL)
}
return c
return c, nil
}
func Record(triggerName string, recorderName string, reqUID string, request *http.Request, originalUrl url.URL, payload string, response *http.Response, namespace string, timestamp int64) {
func Record(logger *zap.Logger, triggerName string, recorderName string, reqUID string, request *http.Request, originalUrl url.URL, payload string, response *http.Response, namespace string, timestamp int64) {
// Case where the function should not have been recorded
if len(reqUID) == 0 {
return
@@ -58,8 +58,9 @@ func Record(triggerName string, recorderName string, reqUID string, request *htt
fullPath := originalUrl.String()
escPayload := string(json.RawMessage(payload))
client := NewClient()
if client == nil {
client, err := NewClient()
if err != nil {
logger.Error("could not create redis client", zap.Error(err))
return
}
@@ -92,7 +93,7 @@ func Record(triggerName string, recorderName string, reqUID string, request *htt
PostForm: postForm,
}
log.Info("Storing request > ", req)
logger.Info("storing request", zap.Any("request", req))
resp := &redisCache.Response{
Status: response.Status,
@@ -107,19 +108,19 @@ func Record(triggerName string, recorderName string, reqUID string, request *htt
data, err := proto.Marshal(ureq)
if err != nil {
log.Error("Error marshalling request: ", err)
logger.Error("error marshalling request", zap.Error(err))
return
}
_, err = client.Do("HMSET", reqUID, "ReqResponse", data, "Timestamp", timestamp, "Trigger", triggerName)
if err != nil {
log.Error("Error saving request: ", err)
logger.Error("error saving request", zap.Error(err))
return
}
_, err = client.Do("LPUSH", recorderName, reqUID)
if err != nil {
log.Error("Error saving recorder-request pair: ", err)
logger.Error("error saving recorder-request pair", zap.Error(err))
return
}
}
redis/redisApi.go
+47 -44
View File
@@ -19,7 +19,6 @@ package redis
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
@@ -27,18 +26,20 @@ import (
"strings"
"time"
"go.uber.org/zap"
"github.com/golang/protobuf/proto"
"github.com/gomodule/redigo/redis"
log "github.com/sirupsen/logrus"
"github.com/pkg/errors"
"github.com/fission/fission/crd"
"github.com/fission/fission/redis/build/gen"
)
func RecordsListAll() ([]byte, error) {
client := NewClient()
if client == nil {
return nil, errors.New("failed to create redis client")
func RecordsListAll(logger *zap.Logger) ([]byte, error) {
client, err := NewClient()
if err != nil {
return nil, errors.Wrap(err, "failed to create redis client")
}
iter := 0
@@ -59,12 +60,12 @@ func RecordsListAll() ([]byte, error) {
if strings.HasPrefix(key, "REQ") {
val, err := redis.Bytes(client.Do("HGET", key, "ReqResponse"))
if err != nil {
log.Error("Error retrieving request from Redis: ", err)
logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err
}
entry, err := deserializeReqResponse(val, key)
if err != nil {
log.Error("Error deserializing request: ", err)
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
filtered = append(filtered, entry)
@@ -84,18 +85,22 @@ func RecordsListAll() ([]byte, error) {
// Input: `from` (hours ago, between 0 [today] and 5) and `to` (same units)
// Note: Fractional values don't seem to work -- document that for the user
func RecordsFilterByTime(from string, to string) ([]byte, error) {
func RecordsFilterByTime(logger *zap.Logger, from string, to string) ([]byte, error) {
rangeStart, rangeEnd, err := obtainInterval(from, to)
log.Debug("Interval inferred: ", rangeStart, rangeEnd)
if rangeStart >= rangeEnd {
log.Error("Invalid chronology")
if err != nil {
return nil, err
}
logger.Debug("interval inferred", zap.Int64("range_start", rangeStart), zap.Int64("range_end", rangeEnd))
client := NewClient()
if rangeStart >= rangeEnd {
e := "invalid chronology - start is greater than or equal to end"
logger.Error(e, zap.Int64("range_start", rangeStart), zap.Int64("range_end", rangeEnd))
return nil, errors.New(e)
}
client, err := NewClient()
if client == nil {
return nil, errors.New("failed to create redis client")
return nil, errors.Wrap(err, "failed to create redis client")
}
iter := 0
@@ -114,24 +119,24 @@ func RecordsFilterByTime(from string, to string) ([]byte, error) {
if strings.HasPrefix(key, "REQ") {
val, err := redis.Strings(client.Do("HMGET", key, "Timestamp"))
if err != nil {
log.Error("Error retrieving timestamp from Redis: ", err)
logger.Error("error retrieving timestamp from redis", zap.Error(err))
return nil, err
}
tsO, err := strconv.Atoi(val[0])
if err != nil {
log.Error("Error converting timestamp to int: ", err)
logger.Error("error converting timestamp to int", zap.Error(err))
return nil, err
}
ts := int64(tsO)
if ts >= rangeStart && ts <= rangeEnd {
val2, err := redis.Bytes(client.Do("HGET", key, "ReqResponse"))
if err != nil {
log.Error("Error retrieving request from Redis: ", err)
logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err
}
entry, err := deserializeReqResponse(val2, key)
if err != nil {
log.Error("Error deserializing request: ", err)
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
filtered = append(filtered, entry)
@@ -151,7 +156,7 @@ func RecordsFilterByTime(from string, to string) ([]byte, error) {
return resp, nil
}
func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) {
func RecordsFilterByTrigger(logger *zap.Logger, queriedTriggerName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) {
matchingRecorders := make(map[string]bool)
// Implicit triggers:
@@ -178,9 +183,9 @@ func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderLi
}
}
client := NewClient()
if client == nil {
return nil, errors.New("failed to create redis client")
client, err := NewClient()
if err != nil {
return nil, errors.Wrap(err, "failed to create redis client")
}
var filtered []*redisCache.RecordedEntry
@@ -195,19 +200,19 @@ func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderLi
for _, reqUID := range val {
val, err := redis.Strings(client.Do("HMGET", reqUID, "Trigger")) // 1-to-1 reqUID - trigger?
if err != nil {
log.Error("Error retrieving trigger for a request from Redis: ", err)
logger.Error("error retrieving trigger for a request from redis", zap.Error(err))
return nil, err
}
if val[0] == queriedTriggerName {
// TODO: Reconsider multiple commands
val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse"))
if err != nil {
log.Error("Error retrieving request from Redis: ", err)
logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err
}
entry, err := deserializeReqResponse(val, reqUID)
if err != nil {
log.Error("Error deserializing request: ", err)
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
filtered = append(filtered, entry)
@@ -222,7 +227,7 @@ func RecordsFilterByTrigger(queriedTriggerName string, recorders *crd.RecorderLi
return resp, nil
}
func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) {
func RecordsFilterByFunction(logger *zap.Logger, queriedFunctionName string, recorders *crd.RecorderList, triggers *crd.HTTPTriggerList) ([]byte, error) {
// Implicit functions:
// Sometimes functions are not explicitly attached to recorders but we still want to be able to
@@ -251,9 +256,9 @@ func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.Recorder
}
}
client := NewClient()
if client == nil {
return nil, errors.New("failed to create redis client")
client, err := NewClient()
if err != nil {
return nil, errors.Wrap(err, "failed to create redis client")
}
var filtered []*redisCache.RecordedEntry
@@ -273,12 +278,12 @@ func RecordsFilterByFunction(queriedFunctionName string, recorders *crd.Recorder
if exists > 0 {
val, err := redis.Bytes(client.Do("HGET", reqUID, "ReqResponse"))
if err != nil {
log.Error("Error retrieving request from Redis: ", err)
logger.Error("error retrieving request from redis", zap.Error(err))
return nil, err
}
entry, err := deserializeReqResponse(val, reqUID)
if err != nil {
log.Error("Error deserializing request: ", err)
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
filtered = append(filtered, entry)
@@ -298,10 +303,8 @@ func deserializeReqResponse(value []byte, reqUID string) (*redisCache.RecordedEn
data := &redisCache.UniqueRequest{}
err := proto.Unmarshal(value, data)
if err != nil {
log.Error("Error unmarshalling request: ", err)
return nil, err
return nil, errors.Wrap(err, "error unmarshalling request")
}
log.Info("Parsed protobuf bytes: ", data)
transformed := &redisCache.RecordedEntry{
ReqUID: reqUID,
Req: data.Req,
@@ -341,38 +344,38 @@ func includesTrigger(triggers []string, query string) bool {
return false
}
func ReplayByReqUID(routerUrl string, queriedID string) ([]byte, error) {
client := NewClient()
if client == nil {
return nil, errors.New("failed to create redis client")
func ReplayByReqUID(logger *zap.Logger, routerUrl string, queriedID string) ([]byte, error) {
client, err := NewClient()
if err != nil {
return nil, errors.Wrap(err, "failed to create redis client")
}
exists, err := redis.Int(client.Do("EXISTS", queriedID))
if exists != 1 || err != nil {
log.Error("couldn't find request to replay")
logger.Error("could not find request to replay in redis", zap.Error(err))
return nil, err
}
val, err := redis.Bytes(client.Do("HGET", queriedID, "ReqResponse"))
if err != nil {
log.Error("couldn't obtain ReqResponse for this ID")
logger.Error("could not obtain ReqResponse for ID from redis", zap.Error(err), zap.String("id", queriedID))
return nil, err
}
entry, err := deserializeReqResponse(val, queriedID)
if err != nil {
log.Error("couldn't deserialize ReqResponse")
logger.Error("error deserializing request from redis", zap.Error(err))
return nil, err
}
replayed, err := ReplayRequest(routerUrl, entry.Req)
if err != nil {
log.Error("couldn't replay request")
logger.Error("error replaying request", zap.Error(err))
return nil, err
}
resp, err := json.Marshal(replayed)
if err != nil {
log.Error("couldn't marshall replayed request response")
logger.Error("error marshalling replayed request response", zap.Error(err))
return nil, err
}
router/functionHandler.go
+48 -24
View File
@@ -33,8 +33,8 @@ import (
"github.com/gorilla/mux"
"github.com/pkg/errors"
"github.com/satori/go.uuid"
log "github.com/sirupsen/logrus"
"go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
@@ -51,6 +51,7 @@ const (
type (
functionHandler struct {
logger *zap.Logger
fmap *functionServiceMap
frmap *functionRecorderMap
trmap *triggerRecorderMap
@@ -86,6 +87,7 @@ type (
// A layer on top of http.DefaultTransport, with retries.
RetryingRoundTripper struct {
logger *zap.Logger
funcHandler *functionHandler
base http.RoundTripper
}
@@ -146,7 +148,7 @@ func (w *fakeCloseReadCloser) RealClose() error {
// if it returned an error.
func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
// Set forwarded host header if not exists
addForwardedHostHeader(req)
roundTripper.addForwardedHostHeader(req)
// TODO: Keep? --> Needed for queries encoded in URL before they're stripped by the proxy
var originalUrl url.URL
@@ -164,7 +166,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
rdr1.Read(p)
postedBody = string(p)
log.Info(fmt.Sprintf("%v", postedBody))
roundTripper.logger.Info("roundtripper posted body", zap.String("body", postedBody))
req.Body = rdr2
}
}
@@ -287,13 +289,15 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
if roundTripper.funcHandler.httpTrigger != nil {
trigger := roundTripper.funcHandler.httpTrigger.Metadata.Name
redis.Record(
roundTripper.logger,
trigger,
roundTripper.funcHandler.recorderName,
req.Header.Get("X-Fission-ReqUID"), req, originalUrl, postedBody, resp, fnMeta.Namespace,
time.Now().UnixNano(),
)
} else {
log.Printf("No http trigger attached for recorder: %v", roundTripper.funcHandler.recorderName)
roundTripper.logger.Error("no http trigger attached for recorder",
zap.String("recorder", roundTripper.funcHandler.recorderName))
}
}
@@ -303,7 +307,7 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
// if transport.RoundTrip returns a non-network dial error, then relay it back to user
if !fission.IsNetworkDialError(err) {
err = errors.Wrapf(err, "Error sending request to function %v", fnMeta.Name)
err = errors.Wrapf(err, "error sending request to function %v", fnMeta.Name)
return resp, err
}
@@ -313,8 +317,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
executingTimeout = executingTimeout * time.Duration(roundTripper.funcHandler.tsRoundTripperParams.timeoutExponent)
retryCounter++
log.Printf("request to %s errored out. backing off for %v before retrying",
req.URL.Host, executingTimeout)
roundTripper.logger.Info("request errored out - backing off before retrying",
zap.String("url", req.URL.Host),
zap.Duration("backoff_timeout", executingTimeout))
time.Sleep(executingTimeout)
@@ -324,9 +329,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
} else {
// if transport.RoundTrip returns a network dial error and serviceUrl was from cache,
// it means, the entry in router cache is stale, so invalidate it.
log.Printf("request to %s errored out. removing function : %s from router's cache "+
"and requesting a new service for function",
req.URL.Host, fnMeta.Name)
roundTripper.logger.Error("request errored out - removing function from router's cache and requesting a new service for function",
zap.String("url", req.URL.Host),
zap.String("function_name", fnMeta.Name))
roundTripper.funcHandler.fmap.remove(fnMeta)
retryCounter = 0
}
@@ -340,7 +345,9 @@ func (roundTripper RetryingRoundTripper) RoundTrip(req *http.Request) (resp *htt
// finally, one more retry with the default timeout
resp, err = http.DefaultTransport.RoundTrip(req)
if err != nil {
log.Printf("Error getting response from function %v: %v", fnMeta.Name, err)
roundTripper.logger.Error("error getting response from function",
zap.Error(err),
zap.String("function_name", fnMeta.Name))
}
return resp, err
@@ -385,19 +392,19 @@ func (fh functionHandler) handler(responseWriter http.ResponseWriter, request *h
UID := strings.ToLower(uuid.NewV4().String())
reqUID = "REQ" + UID
request.Header.Set("X-Fission-ReqUID", reqUID)
log.Print("Record request with ReqUID: ", reqUID)
fh.logger.Info("record request", zap.String("request_id", reqUID))
}
if fh.httpTrigger != nil && fh.httpTrigger.Spec.FunctionReference.Type == fission.FunctionReferenceTypeFunctionWeights {
// canary deployment. need to determine the function to send request to now
fnMetadata := getCanaryBackend(fh.functionMetadataMap, fh.fnWeightDistributionList)
if fnMetadata == nil {
log.Printf("Error getting canary backend ")
fh.logger.Error("could not get canary backend", zap.String("request_id", reqUID))
// TODO : write error to responseWrite and return response
return
}
fh.function = fnMetadata
log.Debugf("chosen fnBackend's metadata : %+v", fh.function)
fh.logger.Debug("chosen function backend's metadata", zap.Any("metadata", fh.function))
}
// system params
@@ -413,6 +420,7 @@ func (fh functionHandler) handler(responseWriter http.ResponseWriter, request *h
proxy := &httputil.ReverseProxy{
Director: director,
Transport: &RetryingRoundTripper{
logger: fh.logger.Named("roundtripper"),
funcHandler: &fh,
base: &ochttp.Transport{
Base: &http.Transport{
@@ -471,7 +479,7 @@ func getCanaryBackend(fnMetadatamap map[string]*metav1.ObjectMeta, fnWtDistribut
}
// addForwardedHostHeader add "forwarded host" to request header
func addForwardedHostHeader(req *http.Request) {
func (roundTripper RetryingRoundTripper) addForwardedHostHeader(req *http.Request) {
// for more detailed information, please visit:
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
@@ -486,7 +494,9 @@ func addForwardedHostHeader(req *http.Request) {
reqUrl := fmt.Sprintf("%s://%s", req.Proto, req.Host)
u, err := url.Parse(reqUrl)
if err != nil {
log.Printf("Error parsing request url (%v): %v", reqUrl, err)
roundTripper.logger.Error("error parsing request url while adding forwarded host headers",
zap.Error(err),
zap.String("url", reqUrl))
return
}
@@ -531,14 +541,19 @@ func (fh *functionHandler) getServiceEntry(ctx context.Context) (serviceUrl *url
var u *url.URL
// Get service entry from executor and update cache if its the first goroutine
if firstToTheLock { // first to the service url
log.Printf("Calling getServiceForFunction for function: %s", fh.function.Name)
fh.logger.Info("calling getServiceForFunction",
zap.String("function_name", fh.function.Name))
u, err = fh.getServiceEntryFromExecutor(ctx)
if err != nil {
log.Printf("Error getting service url from executor: %v", err)
fh.logger.Error("error getting service url from executor",
zap.Error(err),
zap.String("function_name", fh.function.Name))
return nil, err
}
// add the address in router's cache
log.Printf("Assigning service url: %s for function: %s", u, fh.function.Name)
fh.logger.Info("assigning service url for function",
zap.String("url", u.String()),
zap.String("function_name", fh.function.Name))
fh.fmap.assign(fh.function, u)
} else {
u, err = fh.getServiceEntryFromCache()
@@ -554,9 +569,12 @@ func (fh *functionHandler) getServiceEntry(ctx context.Context) (serviceUrl *url
},
)
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("Error updating service address entry for function %v_%v", fh.function.Name, fh.function.Namespace))
log.Println(err)
return nil, false, err
e := "error updating service address entry for function"
fh.logger.Error(e,
zap.Error(err),
zap.String("function_name", fh.function.Name),
zap.String("function_namespace", fh.function.Namespace))
return nil, false, errors.Wrapf(err, "%s %s_%s", e, fh.function.Name, fh.function.Namespace)
}
record, ok := recordObj.(svcEntryRecord)
@@ -596,14 +614,20 @@ func (fh *functionHandler) getServiceEntryFromExecutor(ctx context.Context) (*ur
service, err := fh.executor.GetServiceForFunction(ctx, fh.function)
if err != nil {
statusCode, errMsg := fission.GetHTTPError(err)
log.Printf("Error from GetServiceForFunction for function (%v): %v : %v", fh.function, statusCode, errMsg)
fh.logger.Error("error from GetServiceForFunction",
zap.Error(err),
zap.String("error_message", errMsg),
zap.Any("function", fh.function),
zap.Int("status_code", statusCode))
return nil, err
}
// parse the address into url
serviceUrl, err := url.Parse(fmt.Sprintf("http://%v", service))
if err != nil {
log.Printf("Error parsing service url (%v): %v", serviceUrl, err)
fh.logger.Error("error parsing service url",
zap.Error(err),
zap.String("service_url", serviceUrl.String()))
return nil, err
}
router/functionHandler_test.go
+9 -3
View File
@@ -24,9 +24,11 @@ import (
"testing"
"time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
"github.com/fission/fission/crd"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func createBackendService(testResponseString string) *url.URL {
@@ -53,8 +55,10 @@ func TestFunctionProxying(t *testing.T) {
log.Printf("Created backend svc at %v", backendURL)
fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault}
logger, err := zap.NewDevelopment()
panicIf(err)
fmap := makeFunctionServiceMap(0)
fmap := makeFunctionServiceMap(logger, 0)
fmap.assign(fn, backendURL)
httpTrigger := &crd.HTTPTrigger{
@@ -70,7 +74,9 @@ func TestFunctionProxying(t *testing.T) {
},
}
fh := &functionHandler{fmap: fmap,
fh := &functionHandler{
logger: logger,
fmap: fmap,
function: fn,
tsRoundTripperParams: &tsRoundTripperParams{
timeout: 50 * time.Millisecond,
router/functionRecorderMap.go
+8 -5
View File
@@ -17,9 +17,10 @@ limitations under the License.
package router
import (
"log"
"time"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/cache"
"github.com/fission/fission/crd"
@@ -27,14 +28,16 @@ import (
type (
functionRecorderMap struct {
cache *cache.Cache // map[string]*crd.Recorder
logger *zap.Logger
cache *cache.Cache // map[string]*crd.Recorder
}
)
// Why do we need an expiry?
func makeFunctionRecorderMap(expiry time.Duration) *functionRecorderMap {
func makeFunctionRecorderMap(logger *zap.Logger, expiry time.Duration) *functionRecorderMap {
return &functionRecorderMap{
cache: cache.MakeCache(expiry, 0),
logger: logger.Named("function_recorder_map"),
cache: cache.MakeCache(expiry, 0),
}
}
@@ -53,7 +56,7 @@ func (frmap *functionRecorderMap) assign(function string, recorder *crd.Recorder
if e, ok := err.(fission.Error); ok && e.Code == fission.ErrorNameExists {
return
}
log.Printf("error caching recorder for function name with a different value: %v", err)
frmap.logger.Error("error caching recorder for function name with a different value", zap.Error(err))
}
}
router/functionServiceMap.go
+7 -5
View File
@@ -17,10 +17,10 @@ limitations under the License.
package router
import (
"log"
"net/url"
"time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/cache"
@@ -28,7 +28,8 @@ import (
type (
functionServiceMap struct {
cache *cache.Cache // map[metadataKey]*url.URL
logger *zap.Logger
cache *cache.Cache // map[metadataKey]*url.URL
}
// metav1.ObjectMeta is not hashable, so we make a hashable copy
@@ -40,9 +41,10 @@ type (
}
)
func makeFunctionServiceMap(expiry time.Duration) *functionServiceMap {
func makeFunctionServiceMap(logger *zap.Logger, expiry time.Duration) *functionServiceMap {
return &functionServiceMap{
cache: cache.MakeCache(expiry, 0),
logger: logger.Named("function_service_map"),
cache: cache.MakeCache(expiry, 0),
}
}
@@ -71,7 +73,7 @@ func (fmap *functionServiceMap) assign(f *metav1.ObjectMeta, serviceUrl *url.URL
if *serviceUrl == *(old.(*url.URL)) {
return
}
log.Printf("error caching service url for function with a different value: %v", err)
fmap.logger.Error("error caching service url for function with a different value", zap.Error(err))
// ignore error
}
}
router/functionServiceMap_test.go
+6 -1
View File
@@ -20,11 +20,16 @@ import (
"net/url"
"testing"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestFunctionServiceMap(t *testing.T) {
m := makeFunctionServiceMap(0)
logger, err := zap.NewDevelopment()
panicIf(err)
m := makeFunctionServiceMap(logger, 0)
fn := &metav1.ObjectMeta{Name: "foo", Namespace: metav1.NamespaceDefault}
u, err := url.Parse("/foo012")
if err != nil {
router/httpTriggers.go
+19 -12
View File
@@ -18,11 +18,11 @@ package router
import (
"context"
"log"
"net/http"
"time"
"github.com/gorilla/mux"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/client-go/kubernetes"
@@ -39,6 +39,7 @@ type HTTPTriggerSet struct {
*functionServiceMap
*mutableRouter
logger *zap.Logger
fissionClient *crd.FissionClient
kubeClient *kubernetes.Clientset
executor *executorClient.Client
@@ -57,10 +58,11 @@ type HTTPTriggerSet struct {
svcAddrUpdateThrottler *throttler.Throttler
}
func makeHTTPTriggerSet(fmap *functionServiceMap, frmap *functionRecorderMap, trmap *triggerRecorderMap, fissionClient *crd.FissionClient,
func makeHTTPTriggerSet(logger *zap.Logger, fmap *functionServiceMap, frmap *functionRecorderMap, trmap *triggerRecorderMap, fissionClient *crd.FissionClient,
kubeClient *kubernetes.Clientset, executor *executorClient.Client, crdClient *rest.RESTClient, params *tsRoundTripperParams, isDebugEnv bool, actionThrottler *throttler.Throttler) (*HTTPTriggerSet, k8sCache.Store, k8sCache.Store) {
httpTriggerSet := &HTTPTriggerSet{
logger: logger.Named("http_trigger_set"),
functionServiceMap: fmap,
triggers: []crd.HTTPTrigger{},
fissionClient: fissionClient,
@@ -83,7 +85,7 @@ func makeHTTPTriggerSet(fmap *functionServiceMap, frmap *functionRecorderMap, tr
httpTriggerSet.funcStore = fnStore
httpTriggerSet.funcController = fnController
}
recorderSet = MakeRecorderSet(httpTriggerSet, crdClient, rStore, frmap, trmap)
recorderSet = MakeRecorderSet(logger, httpTriggerSet, crdClient, rStore, frmap, trmap)
httpTriggerSet.recorderSet = recorderSet
return httpTriggerSet, tStore, fnStore
}
@@ -95,7 +97,7 @@ func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter
if ts.fissionClient == nil {
// Used in tests only.
log.Printf("Skipping continuous trigger updates")
ts.logger.Info("skipping continuous trigger updates")
return
}
go ts.updateRouter()
@@ -104,7 +106,7 @@ func (ts *HTTPTriggerSet) subscribeRouter(ctx context.Context, mr *mutableRouter
if ts.recorderSet.recController != nil {
go ts.runWatcher(ctx, ts.recorderSet.recController)
} else {
log.Fatal("Failed to run recorder Controller")
ts.logger.Fatal("failed to run recorder controller")
}
}
@@ -143,10 +145,11 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
if rr.resolveResultType != resolveResultSingleFunction && rr.resolveResultType != resolveResultMultipleFunctions {
// not implemented yet
log.Panicf("resolve result type not implemented (%v)", rr.resolveResultType)
ts.logger.Panic("resolve result type not implemented", zap.Any("type", rr.resolveResultType))
}
fh := &functionHandler{
logger: ts.logger.Named(trigger.Metadata.Name),
fmap: ts.functionServiceMap,
frmap: ts.recorderSet.functionRecorderMap,
trmap: ts.recorderSet.triggerRecorderMap,
@@ -205,6 +208,7 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
}
fh := &functionHandler{
logger: ts.logger.Named(m.Name),
fmap: ts.functionServiceMap,
frmap: ts.recorderSet.functionRecorderMap,
trmap: ts.recorderSet.triggerRecorderMap,
@@ -235,7 +239,7 @@ func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Cont
k8sCache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
trigger := obj.(*crd.HTTPTrigger)
go createIngress(trigger, ts.kubeClient)
go createIngress(ts.logger, trigger, ts.kubeClient)
ts.syncTriggers()
// Check if this trigger's function needs to be recorded
fnRef := trigger.Spec.FunctionReference.Name
@@ -244,14 +248,17 @@ func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Cont
if len(recorder.Spec.Triggers) == 0 {
ts.recorderSet.triggerRecorderMap.assign(trigger.Metadata.Name, recorder)
}
} else if err != nil {
ts.logger.Error("unable to lookup function in functionRecorderMap", zap.Error(err))
} else {
log.Print("Unable to lookup function in functionRecorderMap")
ts.logger.Error("unable to lookup function in functionRecorderMap")
}
},
DeleteFunc: func(obj interface{}) {
ts.syncTriggers()
trigger := obj.(*crd.HTTPTrigger)
go deleteIngress(trigger, ts.kubeClient)
go deleteIngress(ts.logger, trigger, ts.kubeClient)
go ts.recorderSet.DeleteTriggerFromRecorderMap(trigger)
},
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
@@ -262,7 +269,7 @@ func (ts *HTTPTriggerSet) initTriggerController() (k8sCache.Store, k8sCache.Cont
return
}
go updateIngress(oldTrigger, newTrigger, ts.kubeClient)
go updateIngress(ts.logger, oldTrigger, newTrigger, ts.kubeClient)
ts.syncTriggers()
},
})
@@ -296,10 +303,10 @@ func (ts *HTTPTriggerSet) initFunctionController() (k8sCache.Store, k8sCache.Con
rr.functionMetadataMap[fn.Metadata.Name] != nil &&
rr.functionMetadataMap[fn.Metadata.Name].ResourceVersion != fn.Metadata.ResourceVersion {
// invalidate resolver cache
log.Printf("Invalidating resolver cache")
ts.logger.Info("invalidating resolver cache")
err := ts.resolver.delete(key.namespace, key.triggerName, key.triggerResourceVersion)
if err != nil {
log.Printf("Error deleting functionReferenceResolver cache: %v", err)
ts.logger.Error("error deleting functionReferenceResolver cache", zap.Error(err))
}
break
router/ingress.go
+20 -15
View File
@@ -17,9 +17,9 @@ limitations under the License.
package router
import (
"log"
"os"
"go.uber.org/zap"
"k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -38,16 +38,16 @@ func init() {
}
}
func createIngress(trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
func createIngress(logger *zap.Logger, trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
if !trigger.Spec.CreateIngress {
log.Printf("Skipping creation of ingress for trigger: %v", trigger.Metadata.Name)
logger.Info("skipping creation of ingress for trigger", zap.String("trigger", trigger.Metadata.Name))
return
}
_, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{})
if err == nil {
log.Printf("Ingress for trigger exists already %v", trigger.Metadata.Name)
logger.Info("ingress for trigger exists already", zap.String("trigger", trigger.Metadata.Name))
return
}
@@ -87,10 +87,10 @@ func createIngress(trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
_, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Create(ing)
if err != nil {
log.Printf("Failed to create ingress: %v", err)
logger.Error("failed to create ingress", zap.Error(err))
return
}
log.Printf("Created ingress successfully for trigger %v", trigger.Metadata.Name)
logger.Info("created ingress successfully for trigger", zap.String("trigger", trigger.Metadata.Name))
}
func getDeployLabels(trigger *crd.HTTPTrigger) map[string]string {
@@ -101,41 +101,46 @@ func getDeployLabels(trigger *crd.HTTPTrigger) map[string]string {
}
}
func deleteIngress(trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
func deleteIngress(logger *zap.Logger, trigger *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
if !trigger.Spec.CreateIngress {
return
}
ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(trigger.Metadata.Name, v1.GetOptions{})
if err != nil {
log.Printf("Failed to get ingress when deleting trigger: %v, %v", err, trigger)
logger.Error("failed to get ingress when deleting trigger", zap.Error(err), zap.String("trigger", trigger.Metadata.Name))
}
err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Delete(ingress.Name, &v1.DeleteOptions{})
if err != nil {
log.Printf("Failed to delete ingress %v error: %v", ingress, err)
logger.Error("failed to delete ingress for trigger",
zap.Error(err),
zap.Any("ingress", ingress),
zap.String("trigger", trigger.Metadata.Name))
}
}
func updateIngress(oldT *crd.HTTPTrigger, newT *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
func updateIngress(logger *zap.Logger, oldT *crd.HTTPTrigger, newT *crd.HTTPTrigger, kubeClient *kubernetes.Clientset) {
if oldT.Spec.CreateIngress == false && newT.Spec.CreateIngress == true {
createIngress(newT, kubeClient)
createIngress(logger, newT, kubeClient)
return
}
if newT.Spec.CreateIngress == false && oldT.Spec.CreateIngress == true {
deleteIngress(oldT, kubeClient)
deleteIngress(logger, oldT, kubeClient)
return
}
if newT.Spec.Host != oldT.Spec.Host || newT.Spec.RelativeURL != oldT.Spec.RelativeURL {
log.Printf("Updating ingress for trigger %v", oldT.Metadata.Name)
logger.Info("updating ingress for trigger", zap.String("trigger", oldT.Metadata.Name))
ingress, err := kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Get(oldT.Metadata.Name, v1.GetOptions{})
if err != nil {
log.Printf("Failed to get ingress when updating trigger: %v", err)
logger.Error("failed to get ingress when updating trigger",
zap.Error(err),
zap.String("trigger", oldT.Metadata.Name))
}
if newT.Spec.Host != oldT.Spec.Host {
@@ -148,7 +153,7 @@ func updateIngress(oldT *crd.HTTPTrigger, newT *crd.HTTPTrigger, kubeClient *kub
_, err = kubeClient.ExtensionsV1beta1().Ingresses(podNamespace).Update(ingress)
if err != nil {
log.Printf("Failed to update ingress for trigger: %v", err)
logger.Error("failed to update ingress for trigger", zap.String("trigger", oldT.Metadata.Name))
}
}
router/mutablemux.go
+7 -4
View File
@@ -17,11 +17,11 @@ limitations under the License.
package router
import (
"log"
"net/http"
"sync/atomic"
"github.com/gorilla/mux"
"go.uber.org/zap"
)
//
@@ -30,11 +30,14 @@ import (
//
type mutableRouter struct {
logger *zap.Logger
router atomic.Value // mux.Router
}
func NewMutableRouter(handler *mux.Router) *mutableRouter {
mr := mutableRouter{}
func NewMutableRouter(logger *zap.Logger, handler *mux.Router) *mutableRouter {
mr := mutableRouter{
logger: logger.Named("mutable_router"),
}
mr.router.Store(handler)
return &mr
}
@@ -44,7 +47,7 @@ func (mr *mutableRouter) ServeHTTP(responseWriter http.ResponseWriter, request *
routerValue := mr.router.Load()
router, ok := routerValue.(*mux.Router)
if !ok {
log.Panic("Invalid router type")
mr.logger.Panic("invalid router type")
}
router.ServeHTTP(responseWriter, request)
}
router/mutablemux_test.go
+7 -2
View File
@@ -17,11 +17,13 @@ limitations under the License.
package router
import (
"github.com/gorilla/mux"
"log"
"net/http"
"testing"
"time"
"github.com/gorilla/mux"
"go.uber.org/zap"
)
func OldHandler(responseWriter http.ResponseWriter, request *http.Request) {
@@ -62,7 +64,10 @@ func TestMutableMux(t *testing.T) {
log.Print("Create mutable router")
muxRouter := mux.NewRouter()
muxRouter.HandleFunc("/", OldHandler)
mr := NewMutableRouter(muxRouter)
logger, err := zap.NewDevelopment()
panicIf(err)
mr := NewMutableRouter(logger, muxRouter)
// start http server
log.Print("Start http server")
router/recorderController.go
+26 -9
View File
@@ -1,13 +1,16 @@
package router
import (
"github.com/fission/fission/crd"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
"k8s.io/client-go/rest"
k8sCache "k8s.io/client-go/tools/cache"
"github.com/fission/fission/crd"
)
type RecorderSet struct {
logger *zap.Logger
httpTriggerSet *HTTPTriggerSet
crdClient *rest.RESTClient
@@ -19,8 +22,9 @@ type RecorderSet struct {
triggerRecorderMap *triggerRecorderMap
}
func MakeRecorderSet(httpTriggerSet *HTTPTriggerSet, crdClient *rest.RESTClient, rStore k8sCache.Store, frmap *functionRecorderMap, trmap *triggerRecorderMap) *RecorderSet {
func MakeRecorderSet(logger *zap.Logger, httpTriggerSet *HTTPTriggerSet, crdClient *rest.RESTClient, rStore k8sCache.Store, frmap *functionRecorderMap, trmap *triggerRecorderMap) *RecorderSet {
recorderSet := &RecorderSet{
logger: logger.Named("recorder_set"),
httpTriggerSet: httpTriggerSet,
crdClient: crdClient,
recStore: rStore,
@@ -63,12 +67,17 @@ func (rs *RecorderSet) disableRecorder(r *crd.Recorder) {
function := r.Spec.Function
triggers := r.Spec.Triggers
log.Info("Disabling recorder ", r.Metadata.Name)
rs.logger.Info("disabling recorder",
zap.String("recorder", r.Metadata.Name),
zap.String("function", function))
// Account for function
err := rs.functionRecorderMap.remove(function)
if err != nil {
log.Error("Error disabling recorder (failed to remove function from functionRecorderMap): ", err)
rs.logger.Error("error disabling recorder (failed to remove function from functionRecorderMap)",
zap.Error(err),
zap.String("recorder", r.Metadata.Name),
zap.String("function", function))
}
// Account for explicitly added triggers
@@ -76,7 +85,11 @@ func (rs *RecorderSet) disableRecorder(r *crd.Recorder) {
for _, trigger := range triggers {
err := rs.triggerRecorderMap.remove(trigger)
if err != nil {
log.Error("Error disabling recorder (failed to remove triggers from triggerRecorderMap): ", err)
rs.logger.Error("error disabling recorder (failed to remove triggers from triggerRecorderMap)",
zap.Error(err),
zap.String("recorder", r.Metadata.Name),
zap.String("function", function),
zap.String("trigger", trigger))
}
}
} else {
@@ -86,7 +99,11 @@ func (rs *RecorderSet) disableRecorder(r *crd.Recorder) {
if trigger.Spec.FunctionReference.Name == function {
err := rs.triggerRecorderMap.remove(trigger.Metadata.Name)
if err != nil {
log.Error("Failed to remove trigger from triggerRecorderMap: ", err)
rs.logger.Error("failed to remove trigger from triggerRecorderMap",
zap.Error(err),
zap.String("recorder", r.Metadata.Name),
zap.String("function", function),
zap.String("trigger", trigger.Metadata.Name))
}
}
}
@@ -106,13 +123,13 @@ func (rs *RecorderSet) updateRecorder(old *crd.Recorder, newer *crd.Recorder) {
func (rs *RecorderSet) DeleteTriggerFromRecorderMap(trigger *crd.HTTPTrigger) {
err := rs.triggerRecorderMap.remove(trigger.Metadata.Name)
if err != nil {
log.Error("Failed to remove trigger from triggerRecorderMap: ", err)
rs.logger.Error("failed to remove trigger from triggerRecorderMap", zap.Error(err))
}
}
func (rs *RecorderSet) DeleteFunctionFromRecorderMap(function *crd.Function) {
err := rs.functionRecorderMap.remove(function.Metadata.Name)
if err != nil {
log.Error("Failed to remove function from functionRecorderMap: ", err)
rs.logger.Error("failed to remove function from functionRecorderMap", zap.Error(err))
}
}
router/router.go
+57 -32
View File
@@ -42,7 +42,6 @@ package router
import (
"context"
"fmt"
"log"
"net/http"
"os"
"strconv"
@@ -52,6 +51,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opencensus.io/plugin/ochttp"
"go.opencensus.io/trace"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/crd"
@@ -63,16 +63,16 @@ import (
// request url ---[trigger]---> Function(name, deployment) ----[deployment]----> Function(name, uid) ----[pool mgr]---> k8s service url
func router(ctx context.Context, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) *mutableRouter {
func router(ctx context.Context, logger *zap.Logger, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) *mutableRouter {
muxRouter := mux.NewRouter()
mr := NewMutableRouter(muxRouter)
muxRouter.Use(fission.LoggingMiddleware)
mr := NewMutableRouter(logger, muxRouter)
muxRouter.Use(fission.LoggingMiddleware(logger))
httpTriggerSet.subscribeRouter(ctx, mr, resolver)
return mr
}
func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) {
mr := router(ctx, httpTriggerSet, resolver)
func serve(ctx context.Context, logger *zap.Logger, port int, httpTriggerSet *HTTPTriggerSet, resolver *functionReferenceResolver) {
mr := router(ctx, logger, httpTriggerSet, resolver)
url := fmt.Sprintf(":%v", port)
http.ListenAndServe(url, &ochttp.Handler{
Handler: mr,
@@ -82,77 +82,102 @@ func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolv
})
}
func serveMetric() {
func serveMetric(logger *zap.Logger) {
// Expose the registered metrics via HTTP.
http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(metricAddr, nil))
err := http.ListenAndServe(metricAddr, nil)
logger.Fatal("done listening on metrics endpoint", zap.Error(err))
}
func Start(port int, executorUrl string) {
func Start(logger *zap.Logger, port int, executorUrl string) {
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
fmap := makeFunctionServiceMap(time.Minute)
fmap := makeFunctionServiceMap(logger, time.Minute)
frmap := makeFunctionRecorderMap(time.Minute)
frmap := makeFunctionRecorderMap(logger, time.Minute)
trmap := makeTriggerRecorderMap(time.Minute)
trmap := makeTriggerRecorderMap(logger, time.Minute)
fissionClient, kubeClient, _, err := crd.MakeFissionClient()
if err != nil {
log.Fatalf("Error connecting to kubernetes API: %v", err)
logger.Fatal("error connecting to kubernetes API", zap.Error(err))
}
err = fissionClient.WaitForCRDs()
if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err)
logger.Fatal("error waiting for CRDs", zap.Error(err))
}
restClient := fissionClient.GetCrdClient()
executor := executorClient.MakeClient(executorUrl)
executor := executorClient.MakeClient(logger, executorUrl)
timeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_TIMEOUT"))
timeoutStr := os.Getenv("ROUTER_ROUND_TRIP_TIMEOUT")
timeout, err := time.ParseDuration(timeoutStr)
if err != nil {
log.Fatalf("Failed to parse timeout: %v", err)
logger.Fatal("failed to parse timeout duration from 'ROUTER_ROUND_TRIP_TIMEOUT'",
zap.Error(err),
zap.String("value", timeoutStr))
}
timeoutExponent, err := strconv.Atoi(os.Getenv("ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT"))
timeoutExponentStr := os.Getenv("ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT")
timeoutExponent, err := strconv.Atoi(timeoutExponentStr)
if err != nil {
log.Fatalf("Failed to parse timeout exponent: %v", err)
logger.Fatal("failed to parse timeout exponent from 'ROUTER_ROUNDTRIP_TIMEOUT_EXPONENT'",
zap.Error(err),
zap.String("value", timeoutExponentStr))
}
keepAlive, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME"))
keepAliveStr := os.Getenv("ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME")
keepAlive, err := time.ParseDuration(keepAliveStr)
if err != nil {
log.Fatalf("Failed to parse keep alive time: %v", err)
logger.Fatal("failed to parse keep alive duration from 'ROUTER_ROUND_TRIP_KEEP_ALIVE_TIME'",
zap.Error(err),
zap.String("value", keepAliveStr))
}
maxRetries, err := strconv.Atoi(os.Getenv("ROUTER_ROUND_TRIP_MAX_RETRIES"))
maxRetriesStr := os.Getenv("ROUTER_ROUND_TRIP_MAX_RETRIES")
maxRetries, err := strconv.Atoi(maxRetriesStr)
if err != nil {
log.Fatalf("Failed to parse max retry times: %v", err)
logger.Fatal("failed to parse max retries from 'ROUTER_ROUND_TRIP_MAX_RETRIES'",
zap.Error(err),
zap.String("value", maxRetriesStr))
}
isDebugEnv, err := strconv.ParseBool(os.Getenv("DEBUG_ENV"))
isDebugEnvStr := os.Getenv("DEBUG_ENV")
isDebugEnv, err := strconv.ParseBool(isDebugEnvStr)
if err != nil {
log.Fatalf("Failed to parse DEBUG_ENV: %v", err)
logger.Fatal("failed to parse debug env from 'DEBUG_ENV'",
zap.Error(err),
zap.String("value", isDebugEnvStr))
}
// svcAddrRetryCount is the max times for RetryingRoundTripper to retry with a specific service address
svcAddrRetryCount, err := strconv.Atoi(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES"))
svcAddrRetryCountStr := os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES")
svcAddrRetryCount, err := strconv.Atoi(svcAddrRetryCountStr)
if err != nil {
svcAddrRetryCount = 5
log.Printf("Failed to parse svc address retry conunt, set it to default value(5): %v", err)
logger.Info("failed to parse service address retry count from 'ROUTER_ROUND_TRIP_SVC_ADDRESS_MAX_RETRIES' - set to the default value",
zap.Error(err),
zap.String("value", svcAddrRetryCountStr),
zap.Int("default", svcAddrRetryCount))
}
// svcAddrUpdateTimeout is the timeout setting for a goroutine to wait for the update of a service entry.
// If the update process cannot be done within the timeout window, consider it failed.
svcAddrUpdateTimeoutStr := os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT")
svcAddrUpdateTimeout, err := time.ParseDuration(os.Getenv("ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT"))
if err != nil {
svcAddrUpdateTimeout = 30 * time.Second
log.Printf("Failed to parse svc address update timeout, set it to default value(30): %v", err)
logger.Info("failed to parse service address update timeout duration from 'ROUTER_ROUND_TRIP_SVC_ADDRESS_UPDATE_TIMEOUT' - set to the default value",
zap.Error(err),
zap.String("value", svcAddrUpdateTimeoutStr),
zap.Duration("default", svcAddrUpdateTimeout))
}
triggers, _, fnStore := makeHTTPTriggerSet(fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{
triggers, _, fnStore := makeHTTPTriggerSet(logger.Named("triggerset"), fmap, frmap, trmap, fissionClient, kubeClient, executor, restClient, &tsRoundTripperParams{
timeout: timeout,
timeoutExponent: timeoutExponent,
keepAlive: keepAlive,
@@ -162,10 +187,10 @@ func Start(port int, executorUrl string) {
resolver := makeFunctionReferenceResolver(fnStore)
go serveMetric()
go serveMetric(logger)
log.Printf("Starting router at port %v\n", port)
logger.Info("starting router", zap.Int("port", port))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
serve(ctx, port, triggers, resolver)
serve(ctx, logger, port, triggers, resolver)
}
router/router_test.go
+9 -5
View File
@@ -22,6 +22,7 @@ import (
"testing"
"time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
@@ -43,16 +44,19 @@ func TestRouter(t *testing.T) {
testResponseString := "hi"
testServiceUrl := createBackendService(testResponseString)
logger, err := zap.NewDevelopment()
panicIf(err)
// set up the cache with this fake service
fmap := makeFunctionServiceMap(0)
fmap := makeFunctionServiceMap(logger, 0)
fmap.assign(fn, testServiceUrl)
frmap := makeFunctionRecorderMap(time.Minute)
frmap := makeFunctionRecorderMap(logger, time.Minute)
trmap := makeTriggerRecorderMap(time.Minute)
trmap := makeTriggerRecorderMap(logger, time.Minute)
// HTTP trigger set with a trigger for this function
triggers, _, _ := makeHTTPTriggerSet(fmap, frmap, trmap, nil, nil, nil, nil,
triggers, _, _ := makeHTTPTriggerSet(logger, fmap, frmap, trmap, nil, nil, nil, nil,
&tsRoundTripperParams{
timeout: 50 * time.Millisecond,
timeoutExponent: 2,
@@ -95,7 +99,7 @@ func TestRouter(t *testing.T) {
port := 4242
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go serve(ctx, port, triggers, frr)
go serve(ctx, logger, port, triggers, frr)
time.Sleep(100 * time.Millisecond)
// hit the router
router/triggerRecorderMap.go
+8 -5
View File
@@ -17,9 +17,10 @@ limitations under the License.
package router
import (
"log"
"time"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/cache"
"github.com/fission/fission/crd"
@@ -27,13 +28,15 @@ import (
type (
triggerRecorderMap struct {
cache *cache.Cache // map[string]*crd.Recorder
logger *zap.Logger
cache *cache.Cache // map[string]*crd.Recorder
}
)
func makeTriggerRecorderMap(expiry time.Duration) *triggerRecorderMap {
func makeTriggerRecorderMap(logger *zap.Logger, expiry time.Duration) *triggerRecorderMap {
return &triggerRecorderMap{
cache: cache.MakeCache(expiry, 0),
logger: logger.Named("trigger_recorder_map"),
cache: cache.MakeCache(expiry, 0),
}
}
@@ -52,7 +55,7 @@ func (trmap *triggerRecorderMap) assign(trigger string, recorder *crd.Recorder)
if e, ok := err.(fission.Error); ok && e.Code == fission.ErrorNameExists {
return
}
log.Printf("error caching recorder for function name with a different value: %v", err)
trmap.logger.Error("error caching recorder for function name with a different value", zap.Error(err))
}
}
router/util_test.go
+6
View File
@@ -28,3 +28,9 @@ func testRequest(targetUrl string, expectedResponse string) {
log.Panic("Unexpected response")
}
}
func panicIf(err error) {
if err != nil {
log.Panicf("Error: %v", err)
}
}
storagesvc/archivePruner.go
+29 -20
View File
@@ -19,13 +19,14 @@ package storagesvc
import (
"time"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/crd"
)
type ArchivePruner struct {
logger *zap.Logger
crdClient *crd.FissionClient
archiveChan chan (string)
stowClient *StowClient
@@ -34,13 +35,14 @@ type ArchivePruner struct {
const defaultPruneInterval int = 60 // in minutes
func MakeArchivePruner(stowClient *StowClient, pruneInterval time.Duration) (*ArchivePruner, error) {
func MakeArchivePruner(logger *zap.Logger, stowClient *StowClient, pruneInterval time.Duration) (*ArchivePruner, error) {
crdClient, _, _, err := crd.MakeFissionClient()
if err != nil {
return nil, err
}
return &ArchivePruner{
logger: logger.Named("archive_pruner"),
crdClient: crdClient,
archiveChan: make(chan string),
stowClient: stowClient,
@@ -50,15 +52,18 @@ func MakeArchivePruner(stowClient *StowClient, pruneInterval time.Duration) (*Ar
// pruneArchives listens to archiveChannel for archive ids that need to be deleted
func (pruner *ArchivePruner) pruneArchives() {
log.Info("listening to archiveChannel to prune archives")
pruner.logger.Info("listening to archiveChannel to prune archives")
for {
select {
case archiveID := <-pruner.archiveChan:
log.WithField("archive ID", archiveID).Info("sending delete request")
pruner.logger.Info("sending delete request for archive",
zap.String("archive_id", archiveID))
if err := pruner.stowClient.removeFileByID(archiveID); err != nil {
// logging the error and continuing with other deletions.
// hopefully this archive will be deleted in the next iteration.
log.WithField("archiveID", archiveID).WithError(err).Error("Ignoring error while deleting archive")
pruner.logger.Error("ignoring error while deleting archive",
zap.Error(err),
zap.String("archive_id", archiveID))
}
}
}
@@ -72,16 +77,16 @@ func (pruner *ArchivePruner) insertArchive(archiveID string) {
// A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes
// and not the archives that are referenced by them, leaving the archives as orphans.
// getOrphanArchives reaps the orphaned archives.
func (pruner *ArchivePruner) getOrphanArchives() error {
log.Info("getting orphan archives")
func (pruner *ArchivePruner) getOrphanArchives() {
pruner.logger.Info("getting orphan archives")
archivesRefByPkgs := make([]string, 0)
var archiveID string
// get all pkgs from kubernetes
pkgList, err := pruner.crdClient.Packages(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
log.WithError(err).Error("Error getting package list from kubernetes")
return err
pruner.logger.Error("error getting package list from kubernetes", zap.Error(err))
return
}
// extract archives referenced by these pkgs
@@ -89,44 +94,48 @@ func (pruner *ArchivePruner) getOrphanArchives() error {
if pkg.Spec.Deployment.URL != "" {
archiveID, err = getQueryParamValue(pkg.Spec.Deployment.URL, "id")
if err != nil {
log.WithError(err).Error("Error extracting value of archiveID from url")
return err
pruner.logger.Error("error extracting value of archiveID from deployment url",
zap.Error(err),
zap.String("url", pkg.Spec.Deployment.URL))
return
}
archivesRefByPkgs = append(archivesRefByPkgs, archiveID)
}
if pkg.Spec.Source.URL != "" {
archiveID, err = getQueryParamValue(pkg.Spec.Source.URL, "id")
if err != nil {
log.WithError(err).Error("Error extracting value of archiveID from url")
return err
pruner.logger.Error("error extracting value of archiveID from source url",
zap.Error(err),
zap.String("url", pkg.Spec.Source.URL))
return
}
archivesRefByPkgs = append(archivesRefByPkgs, archiveID)
}
}
log.WithField("list", "archives referenced by packages").Debugf("%s", archivesRefByPkgs)
pruner.logger.Debug("archives referenced by packagese", zap.Strings("archives", archivesRefByPkgs))
// get all archives on storage
// out of them, there may be some just created but not referenced by packages yet.
// need to filter them out.
archivesInStorage, err := pruner.stowClient.getItemIDsWithFilter(filterItemCreatedAMinuteAgo, time.Now())
archivesInStorage, err := pruner.stowClient.getItemIDsWithFilter(pruner.stowClient.filterItemCreatedAMinuteAgo, time.Now())
if err != nil {
log.WithError(err).Error("Error getting items from storage")
return err
pruner.logger.Error("error getting items from storage", zap.Error(err))
return
}
log.WithField("list", "archives in storage").Debugf("%s", archivesInStorage)
pruner.logger.Debug("archives in storage", zap.Strings("archives", archivesInStorage))
// difference of the two lists gives us the list of orphan archives. This is just a brute force approach.
// need to do something more optimal at scale.
orphanedArchives := getDifferenceOfLists(archivesInStorage, archivesRefByPkgs)
log.WithField("list", "orphan archives").Debugf("%s", orphanedArchives)
pruner.logger.Debug("orphan archives", zap.Strings("archives", orphanedArchives))
// send each orphan archive away for deletion
for _, archiveID = range orphanedArchives {
pruner.insertArchive(archiveID)
}
return nil
return
}
// Start starts a go routine that listens to a channel for archive IDs that need to deleted.
storagesvc/client/storagesvc_test.go
+5 -1
View File
@@ -27,6 +27,7 @@ import (
"time"
"github.com/dchest/uniuri"
"go.uber.org/zap"
"github.com/fission/fission/storagesvc"
)
@@ -52,9 +53,12 @@ func TestStorageService(t *testing.T) {
port := 8080
enableArchivePruner := false
logger, err := zap.NewDevelopment()
panicIf(err)
log.Println("starting storage svc")
_ = storagesvc.RunStorageService(
storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner)
logger, storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner)
time.Sleep(time.Second)
client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))
storagesvc/storagesvc.go
+30 -20
View File
@@ -25,15 +25,17 @@ import (
"strconv"
"time"
"github.com/fission/fission"
"github.com/gorilla/mux"
_ "github.com/graymeta/stow/local"
log "github.com/sirupsen/logrus"
"go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
"github.com/fission/fission"
)
type (
StorageService struct {
logger *zap.Logger
storageClient *StowClient
port int
}
@@ -61,27 +63,34 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request)
fileSizeS, ok := r.Header["X-File-Size"]
if !ok {
log.Error("Missing X-File-Size")
ss.logger.Error("upload is missing the 'X-File-Size' header",
zap.String("filename", handler.Filename))
http.Error(w, "missing X-File-Size header", http.StatusBadRequest)
return
}
fileSize, err := strconv.Atoi(fileSizeS[0])
if err != nil {
log.WithError(err).Errorf("Error parsing x-file-size: '%v'", fileSizeS)
ss.logger.Error("error parsing 'X-File-Size' header",
zap.Error(err),
zap.Strings("header", fileSizeS),
zap.String("filename", handler.Filename))
http.Error(w, "missing or bad X-File-Size header", http.StatusBadRequest)
return
}
// TODO: allow headers to add more metadata (e.g. environment
// and function metadata)
log.Infof("Handling upload for %v", handler.Filename)
ss.logger.Info("handling upload",
zap.String("filename", handler.Filename))
//fileMetadata := make(map[string]interface{})
//fileMetadata["filename"] = handler.Filename
id, err := ss.storageClient.putFile(file, int64(fileSize))
if err != nil {
log.WithError(err).Error("Error saving uploaded file")
ss.logger.Error("error saving uploaded file",
zap.Error(err),
zap.String("filename", handler.Filename))
http.Error(w, "Error saving uploaded file", http.StatusInternalServerError)
return
}
@@ -92,6 +101,9 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request)
}
resp, err := json.Marshal(ur)
if err != nil {
ss.logger.Error("error marshaling uploaded file response",
zap.Error(err),
zap.String("filename", handler.Filename))
http.Error(w, "Error marshaling response", http.StatusInternalServerError)
return
}
@@ -136,7 +148,7 @@ func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request
// stream it to response
err = ss.storageClient.copyFileToStream(fileId, w)
if err != nil {
log.WithError(err).Errorf("Error getting item id '%v'", fileId)
ss.logger.Error("error getting file from storage client", zap.Error(err), zap.String("file_id", fileId))
if err == ErrNotFound {
http.Error(w, "Error retrieving item: not found", http.StatusNotFound)
} else if err == ErrRetrievingItem {
@@ -154,8 +166,9 @@ func (ss *StorageService) healthHandler(w http.ResponseWriter, r *http.Request)
w.WriteHeader(http.StatusOK)
}
func MakeStorageService(storageClient *StowClient, port int) *StorageService {
func MakeStorageService(logger *zap.Logger, storageClient *StowClient, port int) *StorageService {
return &StorageService{
logger: logger.Named("storage_service"),
storageClient: storageClient,
port: port,
}
@@ -170,30 +183,27 @@ func (ss *StorageService) Start(port int) {
address := fmt.Sprintf(":%v", port)
r.Use(fission.LoggingMiddleware)
r.Use(fission.LoggingMiddleware(ss.logger))
err := http.ListenAndServe(address, &ochttp.Handler{
Handler: r,
// Propagation: &b3.HTTPFormat{},
})
log.Fatal(err)
ss.logger.Fatal("done listening", zap.Error(err))
}
func RunStorageService(storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService {
func RunStorageService(logger *zap.Logger, storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService {
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
// initialize logger
log.SetLevel(log.InfoLevel)
// create a storage client
storageClient, err := MakeStowClient(storageType, storagePath, containerName)
storageClient, err := MakeStowClient(logger, storageType, storagePath, containerName)
if err != nil {
log.Fatalf("Error creating stowClient: %v", err)
logger.Fatal("error creating stowClient", zap.Error(err))
}
// create http handlers
storageService := MakeStorageService(storageClient, port)
storageService := MakeStorageService(logger, storageClient, port)
go storageService.Start(port)
// enablePruner prevents storagesvc unit test from needing to talk to kubernetes
@@ -203,13 +213,13 @@ func RunStorageService(storageType StorageType, storagePath string, containerNam
if err != nil {
pruneInterval = defaultPruneInterval
}
pruner, err := MakeArchivePruner(storageClient, time.Duration(pruneInterval))
pruner, err := MakeArchivePruner(logger, storageClient, time.Duration(pruneInterval))
if err != nil {
log.Fatalf("Error creating archivePruner: %v", err)
logger.Fatal("error creating archivePruner", zap.Error(err))
}
go pruner.Start()
}
log.Info("Storage service started")
logger.Info("storage service started")
return storageService
}
storagesvc/stowClient.go
+16 -11
View File
@@ -17,7 +17,6 @@ limitations under the License.
package storagesvc
import (
"errors"
"io"
"mime/multipart"
"os"
@@ -25,8 +24,9 @@ import (
"github.com/graymeta/stow"
_ "github.com/graymeta/stow/local"
"github.com/pkg/errors"
"github.com/satori/go.uuid"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
)
type (
@@ -40,6 +40,7 @@ type (
}
StowClient struct {
logger *zap.Logger
config *storageConfig
location stow.Location
container stow.Container
@@ -59,7 +60,7 @@ var (
ErrWritingFileIntoResponse = errors.New("unable to copy item into http response")
)
func MakeStowClient(storageType StorageType, storagePath string, containerName string) (*StowClient, error) {
func MakeStowClient(logger *zap.Logger, storageType StorageType, storagePath string, containerName string) (*StowClient, error) {
if storageType != StorageTypeLocal {
return nil, errors.New("Storage types other than 'local' are not implemented")
}
@@ -71,13 +72,13 @@ func MakeStowClient(storageType StorageType, storagePath string, containerName s
}
stowClient := &StowClient{
logger: logger.Named("stow_client"),
config: config,
}
cfg := stow.ConfigMap{"path": config.localPath}
loc, err := stow.Dial("local", cfg)
if err != nil {
log.WithError(err).Error("Error initializing storage")
return nil, err
}
stowClient.location = loc
@@ -99,7 +100,6 @@ func MakeStowClient(storageType StorageType, storagePath string, containerName s
}
}
if err != nil {
log.WithError(err).Error("Error initializing storage")
return nil, err
}
stowClient.container = con
@@ -116,11 +116,13 @@ func (client *StowClient) putFile(file multipart.File, fileSize int64) (string,
// save the file to the storage backend
item, err := client.container.Put(uploadName, file, int64(fileSize), nil)
if err != nil {
log.WithError(err).Errorf("Error writing file: %s on storage", uploadName)
client.logger.Error("error writing file on storage",
zap.Error(err),
zap.String("file", uploadName))
return "", ErrWritingFile
}
log.Debugf("Successfully wrote file:%s on storage", uploadName)
client.logger.Debug("successfully wrote file on storage", zap.String("file", uploadName))
return item.ID(), nil
}
@@ -146,7 +148,7 @@ func (client *StowClient) copyFileToStream(fileId string, w io.Writer) error {
return ErrWritingFileIntoResponse
}
log.Debugf("successfully wrote file: %s into httpresponse", fileId)
client.logger.Debug("successfully wrote file into httpresponse", zap.String("file", fileId))
return nil
}
@@ -169,7 +171,7 @@ func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncPara
for {
items, cursor, err = client.container.Items(stow.NoPrefix, cursor, PaginationSize)
if err != nil {
log.WithError(err).Error("Error getting items from container")
errors.Wrap(err, "error getting items from container")
return nil, err
}
@@ -191,10 +193,13 @@ func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncPara
// filterItemCreatedAMinuteAgo is one type of filter function that filters out items created less than a minute ago.
// More filter functions can be written if needed, as long as they are of type filter
func filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool {
func (client StowClient) filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool {
itemLastModTime, _ := item.LastMod()
if currentTime.(time.Time).Sub(itemLastModTime) < 1*time.Minute {
log.Debugf("item: %s created less than a minute ago: %v", item.ID(), itemLastModTime)
client.logger.Debug("item created less than a minute ago",
zap.String("item", item.ID()),
zap.Time("last_modified_time", itemLastModTime))
return true
}
return false
storagesvc/util.go
+3 -3
View File
@@ -17,15 +17,15 @@ limitations under the License.
package storagesvc
import (
"log"
"net/url"
"github.com/pkg/errors"
)
func getQueryParamValue(urlString string, queryParam string) (string, error) {
url, err := url.Parse(urlString)
if err != nil {
log.Printf("Error parsing URL string: %s into URL", urlString)
return "", err
return "", errors.Wrapf(err, "error parsing URL string %q into URL", urlString)
}
return url.Query().Get(queryParam), nil
}
test/test_utils.sh
+1 -1
View File
@@ -193,7 +193,7 @@ helm_install_fission() {
ns=f-$id
fns=f-func-$id
helmVars=repository=$repo,image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImage=$fluentdImage,logger.fluentdImageTag=$fluentdImageTag,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType,serviceType=$serviceType,preUpgradeChecksImage=$preUpgradeCheckImage,prometheus.server.persistentVolume.enabled=false,prometheus.alertmanager.enabled=false,prometheus.kubeStateMetrics.enabled=false,prometheus.nodeExporter.enabled=false
helmVars=repository=$repo,image=$image,imageTag=$imageTag,fetcherImage=$fetcherImage,fetcherImageTag=$fetcherImageTag,functionNamespace=$fns,controllerPort=$controllerNodeport,routerPort=$routerNodeport,pullPolicy=Always,analytics=false,logger.fluentdImageRepository=$repo,logger.fluentdImage=$fluentdImage,logger.fluentdImageTag=$fluentdImageTag,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType,serviceType=$serviceType,preUpgradeChecksImage=$preUpgradeCheckImage,prometheus.server.persistentVolume.enabled=false,prometheus.alertmanager.enabled=false,prometheus.kubeStateMetrics.enabled=false,prometheus.nodeExporter.enabled=false
timeout 30 bash -c "helm_setup"
test/upgrade/fission_upgrade_test.sh
+1 -1
View File
@@ -93,7 +93,7 @@ sudo mv $ROOT/fission/fission /usr/local/bin/
## Upgrade
helmVars=repository=$repo,image=$IMAGE,imageTag=$TAG,fetcherImage=$FETCHER_IMAGE,fetcherImageTag=$TAG,logger.fluentdImage=$FLUENTD_IMAGE,logger.fluentdImageTag=$TAG,functionNamespace=$fns,controllerPort=$controllerNodeport,pullPolicy=Always,analytics=false,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType
helmVars=repository=$repo,image=$IMAGE,imageTag=$TAG,fetcherImage=$FETCHER_IMAGE,fetcherImageTag=$TAG,logger.fluentdImageRepository=$repo,logger.fluentdImage=$FLUENTD_IMAGE,logger.fluentdImageTag=$TAG,functionNamespace=$fns,controllerPort=$controllerNodeport,pullPolicy=Always,analytics=false,pruneInterval=$pruneInterval,routerServiceType=$routerServiceType
echo "Upgrading fission"
helm upgrade \
timer/main.go
+7 -6
View File
@@ -17,25 +17,26 @@ limitations under the License.
package timer
import (
"log"
"github.com/pkg/errors"
"go.uber.org/zap"
"github.com/fission/fission/crd"
"github.com/fission/fission/publisher"
)
func Start(routerUrl string) error {
func Start(logger *zap.Logger, routerUrl string) error {
fissionClient, _, _, err := crd.MakeFissionClient()
if err != nil {
return err
return errors.Wrap(err, "failed to get fission or kubernetes client")
}
err = fissionClient.WaitForCRDs()
if err != nil {
log.Fatalf("Error waiting for CRDs: %v", err)
return errors.Wrap(err, "error waiting for CRDs")
}
poster := publisher.MakeWebhookPublisher(routerUrl)
MakeTimerSync(fissionClient, MakeTimer(poster))
poster := publisher.MakeWebhookPublisher(logger, routerUrl)
MakeTimerSync(logger, fissionClient, MakeTimer(logger, poster))
return nil
}
timer/timer.go
+6 -5
View File
@@ -17,9 +17,8 @@ limitations under the License.
package timer
import (
"log"
"github.com/robfig/cron"
"go.uber.org/zap"
"github.com/fission/fission"
"github.com/fission/fission/crd"
@@ -34,6 +33,7 @@ const (
type (
Timer struct {
logger *zap.Logger
triggers map[string]*timerTriggerWithCron
requestChannel chan *timerRequest
publisher *publisher.Publisher
@@ -53,8 +53,9 @@ type (
}
)
func MakeTimer(publisher publisher.Publisher) *Timer {
func MakeTimer(logger *zap.Logger, publisher publisher.Publisher) *Timer {
timer := &Timer{
logger: logger.Named("timer"),
triggers: make(map[string]*timerTriggerWithCron),
requestChannel: make(chan *timerRequest),
publisher: &publisher,
@@ -114,7 +115,7 @@ func (timer *Timer) syncCron(triggers []crd.TimeTrigger) error {
if _, found := triggerMap[k]; !found {
if v.cron != nil {
v.cron.Stop()
log.Printf("Cron for time trigger %s stopped", v.trigger.Metadata.Name)
timer.logger.Info("cron for time trigger stopped", zap.String("trigger", v.trigger.Metadata.Name))
}
delete(timer.triggers, k)
}
@@ -136,6 +137,6 @@ func (timer *Timer) newCron(t crd.TimeTrigger) *cron.Cron {
(*timer.publisher).Publish("", headers, fission.UrlForFunction(t.Spec.FunctionReference.Name, t.Metadata.Namespace))
})
c.Start()
log.Printf("Add new cron for time trigger %v", t.Metadata.Name)
timer.logger.Info("added new cron for time trigger", zap.String("trigger", t.Metadata.Name))
return c
}
timer/timerSync.go
+6 -4
View File
@@ -17,9 +17,9 @@ limitations under the License.
package timer
import (
"log"
"time"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission"
@@ -28,13 +28,15 @@ import (
type (
TimerSync struct {
logger *zap.Logger
fissionClient *crd.FissionClient
timer *Timer
}
)
func MakeTimerSync(fissionClient *crd.FissionClient, timer *Timer) *TimerSync {
func MakeTimerSync(logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync {
ws := &TimerSync{
logger: logger.Named("timer_sync"),
fissionClient: fissionClient,
timer: timer,
}
@@ -47,11 +49,11 @@ func (ws *TimerSync) syncSvc() {
triggers, err := ws.fissionClient.TimeTriggers(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
if fission.IsNetworkError(err) {
log.Printf("Encounter network error, retry again: %v", err)
ws.logger.Info("encountered a network error - will retry", zap.Error(err))
time.Sleep(5 * time.Second)
continue
}
log.Fatalf("Failed get time trigger list: %v", err)
ws.logger.Fatal("failed to get time trigger list", zap.Error(err))
}
ws.timer.Sync(triggers.Items)