Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Capture os signals to gracefully shutdown fission components (#2261)

Browse Source 
- Currently, fission components don't handle shutdown signals.
So we don't get any to do the required cleanup before the fission process
exits. Adding signal capture process with cancelling context so
that all dependent processes stop working when the process gets term
signal.
- Set log level to error in otel shutdown function

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Sanket Sudake
2021-11-11 17:02:11 +05:30
committed by GitHub
parent d260f07acb
commit 59e876062f
26 changed files with 161 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/fetcher/app/server.go
+1 -3
View File
@@ -39,7 +39,7 @@ var (
readyToServe uint32
)
func Run(logger *zap.Logger) {
func Run(ctx context.Context, logger *zap.Logger) {
flag.Usage = fetcherUsage
collectorEndpoint := flag.String("jaeger-collector-endpoint", "", "")
specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup")
@@ -62,8 +62,6 @@ func Run(logger *zap.Logger) {
}
}
}
ctx := context.Background()
openTracingEnabled := tracing.TracingEnabled(logger)
if openTracingEnabled {
go func() {
cmd/fetcher/main.go
+3 -1
View File
@@ -20,6 +20,7 @@ import (
"github.com/fission/fission/cmd/fetcher/app"
"github.com/fission/fission/pkg/utils/loggerfactory"
"github.com/fission/fission/pkg/utils/profile"
"github.com/fission/fission/pkg/utils/signals"
)
// Usage: fetcher <shared volume path>
@@ -29,5 +30,6 @@ func main() {
profile.ProfileIfEnabled(logger)
app.Run(logger)
ctx := signals.SetupSignalHandlerWithContext(logger)
app.Run(ctx, logger)
}
cmd/fission-bundle/main.go
+35 -33
View File
@@ -41,27 +41,28 @@ import (
"github.com/fission/fission/pkg/utils/loggerfactory"
"github.com/fission/fission/pkg/utils/otel"
"github.com/fission/fission/pkg/utils/profile"
"github.com/fission/fission/pkg/utils/signals"
"github.com/fission/fission/pkg/utils/tracing"
)
func runController(logger *zap.Logger, port int, openTracingEnabled bool) {
controller.Start(logger, port, false, openTracingEnabled)
func runController(ctx context.Context, logger *zap.Logger, port int, openTracingEnabled bool) {
controller.Start(ctx, logger, port, false, openTracingEnabled)
}
func runRouter(logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) {
router.Start(logger, port, executorUrl, openTracingEnabled)
func runRouter(ctx context.Context, logger *zap.Logger, port int, executorUrl string, openTracingEnabled bool) {
router.Start(ctx, logger, port, executorUrl, openTracingEnabled)
}
func runExecutor(logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error {
return executor.StartExecutor(logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled)
func runExecutor(ctx context.Context, logger *zap.Logger, port int, functionNamespace, envBuilderNamespace string, openTracingEnabled bool) error {
return executor.StartExecutor(ctx, logger, functionNamespace, envBuilderNamespace, port, openTracingEnabled)
}
func runKubeWatcher(logger *zap.Logger, routerUrl string) error {
return kubewatcher.Start(logger, routerUrl)
func runKubeWatcher(ctx context.Context, logger *zap.Logger, routerUrl string) error {
return kubewatcher.Start(ctx, logger, routerUrl)
}
func runTimer(logger *zap.Logger, routerUrl string) error {
return timer.Start(logger, routerUrl)
func runTimer(ctx context.Context, logger *zap.Logger, routerUrl string) error {
return timer.Start(ctx, logger, routerUrl)
}
func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error {
@@ -69,20 +70,20 @@ func runMessageQueueMgr(logger *zap.Logger, routerUrl string) error {
}
// KEDA based MessageQueue Trigger Manager
func runMQManager(logger *zap.Logger, routerURL string) error {
return mqt.StartScalerManager(logger, routerURL)
func runMQManager(ctx context.Context, logger *zap.Logger, routerURL string) error {
return mqt.StartScalerManager(ctx, logger, routerURL)
}
func runStorageSvc(logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error {
return storagesvc.Start(logger, storage, port, openTracingEnabled)
func runStorageSvc(ctx context.Context, logger *zap.Logger, port int, storage storagesvc.Storage, openTracingEnabled bool) error {
return storagesvc.Start(ctx, logger, storage, port, openTracingEnabled)
}
func runBuilderMgr(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
return buildermgr.Start(logger, storageSvcUrl, envBuilderNamespace)
func runBuilderMgr(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
return buildermgr.Start(ctx, logger, storageSvcUrl, envBuilderNamespace)
}
func runLogger() {
functionLogger.Start()
func runLogger(ctx context.Context, logger *zap.Logger) {
functionLogger.Start(ctx, logger)
}
func getPort(logger *zap.Logger, portArg interface{}) int {
@@ -129,10 +130,8 @@ func getServiceName(arguments map[string]interface{}) string {
}
func exitWithSync(logger *zap.Logger) {
err := logger.Sync()
if err != nil {
logger.Error("failed to sync log", zap.Error(err))
}
// Ignore error, safe to ignore as per https://github.com/uber-go/zap/issues/328
_ = logger.Sync()
os.Exit(1)
}
@@ -210,7 +209,8 @@ Options:
return
}
ctx := context.Background()
ctx := signals.SetupSignalHandlerWithContext(logger)
openTracingEnabled := tracing.TracingEnabled(logger)
if openTracingEnabled {
err = tracing.RegisterTraceExporter(logger, os.Getenv("TRACE_JAEGER_COLLECTOR_ENDPOINT"), getServiceName(arguments))
@@ -238,21 +238,21 @@ Options:
if arguments["--controllerPort"] != nil {
port := getPort(logger, arguments["--controllerPort"])
runController(logger, port, openTracingEnabled)
runController(ctx, logger, port, openTracingEnabled)
logger.Error("controller exited")
return
}
if arguments["--routerPort"] != nil {
port := getPort(logger, arguments["--routerPort"])
runRouter(logger, port, executorUrl, openTracingEnabled)
runRouter(ctx, logger, port, executorUrl, openTracingEnabled)
logger.Error("router exited")
return
}
if arguments["--executorPort"] != nil {
port := getPort(logger, arguments["--executorPort"])
err = runExecutor(logger, port, functionNs, envBuilderNs, openTracingEnabled)
err = runExecutor(ctx, logger, port, functionNs, envBuilderNs, openTracingEnabled)
if err != nil {
logger.Error("executor exited", zap.Error(err))
return
@@ -260,7 +260,7 @@ Options:
}
if arguments["--kubewatcher"] == true {
err = runKubeWatcher(logger, routerUrl)
err = runKubeWatcher(ctx, logger, routerUrl)
if err != nil {
logger.Error("kubewatcher exited", zap.Error(err))
return
@@ -268,7 +268,7 @@ Options:
}
if arguments["--timer"] == true {
err = runTimer(logger, routerUrl)
err = runTimer(ctx, logger, routerUrl)
if err != nil {
logger.Error("timer exited", zap.Error(err))
return
@@ -284,7 +284,7 @@ Options:
}
if arguments["--mqt_keda"] == true {
err = runMQManager(logger, routerUrl)
err = runMQManager(ctx, logger, routerUrl)
if err != nil {
logger.Error("mqt scaler manager exited", zap.Error(err))
return
@@ -292,7 +292,7 @@ Options:
}
if arguments["--builderMgr"] == true {
err = runBuilderMgr(logger, storageSvcUrl, envBuilderNs)
err = runBuilderMgr(ctx, logger, storageSvcUrl, envBuilderNs)
if err != nil {
logger.Error("builder manager exited", zap.Error(err))
return
@@ -300,7 +300,7 @@ Options:
}
if arguments["--logger"] == true {
runLogger()
runLogger(ctx, logger)
logger.Error("logger exited")
return
}
@@ -315,11 +315,13 @@ Options:
} else if arguments["--storageType"] == string(storagesvc.StorageTypeLocal) {
storage = storagesvc.NewLocalStorage("/fission")
}
err := runStorageSvc(logger, port, storage, openTracingEnabled)
err := runStorageSvc(ctx, logger, port, storage, openTracingEnabled)
if err != nil {
logger.Error("storage service exited", zap.Error(err))
return
}
}
select {}
<-ctx.Done()
logger.Error("exiting")
}
cmd/preupgradechecks/main.go
+2 -3
View File
@@ -17,13 +17,12 @@ limitations under the License.
package main
import (
"context"
"github.com/docopt/docopt-go"
"go.uber.org/zap"
"github.com/fission/fission/pkg/info"
"github.com/fission/fission/pkg/utils/loggerfactory"
"github.com/fission/fission/pkg/utils/signals"
)
func getStringArgWithDefault(arg interface{}, defaultValue string) string {
@@ -59,7 +58,7 @@ Options:
zap.Error(err))
}
ctx := context.Background()
ctx := signals.SetupSignalHandlerWithContext(logger)
crd := crdBackedClient.GetFunctionCRD(ctx)
if crd == nil {
logger.Info("nothing to do since CRDs are not present on the cluster")
cmd/reporter/app/cmd_event.go
+3 -1
View File
@@ -16,6 +16,7 @@ limitations under the License.
package app
import (
"context"
"log"
"github.com/spf13/cobra"
@@ -49,7 +50,8 @@ func eventCommandHandler(cmd *cobra.Command, args []string) error {
return err
}
return tracker.Tracker.SendEvent(event)
ctx := context.Background()
return tracker.Tracker.SendEvent(ctx, event)
}
//EventCommand reports an event to analytics
pkg/buildermgr/buildermgr.go
+3 -2
View File
@@ -17,6 +17,7 @@ limitations under the License.
package buildermgr
import (
"context"
"time"
"github.com/pkg/errors"
@@ -29,7 +30,7 @@ import (
)
// Start the buildermgr service.
func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
func Start(ctx context.Context, logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string) error {
bmLogger := logger.Named("builder_manager")
fissionClient, kubernetesClient, _, _, err := crd.MakeFissionClient()
@@ -56,6 +57,6 @@ func Start(logger *zap.Logger, storageSvcUrl string, envBuilderNamespace string)
pkgInformer := informerFactory.Core().V1().Packages().Informer()
pkgWatcher := makePackageWatcher(bmLogger, fissionClient,
kubernetesClient, envBuilderNamespace, storageSvcUrl, &podInformer, &pkgInformer)
pkgWatcher.Run()
pkgWatcher.Run(ctx)
return nil
}
pkg/buildermgr/pkgwatcher.go
+3 -4
View File
@@ -322,11 +322,10 @@ func (pkgw *packageWatcher) packageInformerHandler() k8sCache.ResourceEventHandl
}
}
func (pkgw *packageWatcher) Run() {
context := context.Background()
go (*pkgw.podInformer).Run(context.Done())
func (pkgw *packageWatcher) Run(ctx context.Context) {
go (*pkgw.podInformer).Run(ctx.Done())
(*pkgw.pkgInformer).AddEventHandler(pkgw.packageInformerHandler())
(*pkgw.pkgInformer).Run(context.Done())
(*pkgw.pkgInformer).Run(ctx.Done())
}
// setInitialBuildStatus sets initial build status to a package if it is empty.
pkg/controller/api_test.go
+2 -1
View File
@@ -376,7 +376,8 @@ func TestMain(m *testing.M) {
panicIf(err)
go Start(logger, 8888, true, true)
ctx := context.Background()
go Start(ctx, logger, 8888, true, true)
time.Sleep(5 * time.Second)
pkg/controller/controller.go
+1 -3
View File
@@ -24,7 +24,7 @@ import (
"github.com/fission/fission/pkg/crd"
)
func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) {
func Start(ctx context.Context, logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled bool) {
cLogger := logger.Named("controller")
fc, kc, apiExtClient, _, err := crd.MakeFissionClient()
@@ -42,12 +42,10 @@ func Start(logger *zap.Logger, port int, unitTestFlag bool, openTracingEnabled b
cLogger.Fatal("error waiting for CRDs", zap.Error(err))
}
ctx, cancel := context.WithCancel(context.Background())
featureStatus, err := ConfigureFeatures(ctx, cLogger, unitTestFlag, fc, kc)
if err != nil {
cLogger.Error("error configuring features - proceeding without optional features", zap.Error(err))
}
defer cancel()
api, err := MakeAPI(cLogger, featureStatus)
if err != nil {
pkg/executor/executor.go
+2 -3
View File
@@ -261,7 +261,7 @@ func serveMetric(logger *zap.Logger) {
// StartExecutor Starts executor and the executor components such as Poolmgr,
// deploymgr and potential future executor types
func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error {
func StartExecutor(ctx context.Context, logger *zap.Logger, functionNamespace string, envBuilderNamespace string, port int, openTracingEnabled bool) error {
fissionClient, kubernetesClient, _, metricsClient, err := crd.MakeFissionClient()
if err != nil {
return errors.Wrap(err, "failed to get kubernetes client")
@@ -324,7 +324,6 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames
}
cnmDeplInformer := cnmInformerFactory.Apps().V1().Deployments()
cnmSvcInformer := cnmInformerFactory.Core().V1().Services()
ctx := context.Background()
cnm, err := container.MakeContainer(
ctx, logger,
fissionClient, kubernetesClient,
@@ -379,7 +378,7 @@ func StartExecutor(logger *zap.Logger, functionNamespace string, envBuilderNames
if err != nil {
return err
}
go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
go reaper.CleanupRoleBindings(ctx, logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
go api.Serve(port, openTracingEnabled)
go serveMetric(logger)
pkg/executor/executor_test.go
+2 -1
View File
@@ -173,7 +173,8 @@ func TestExecutor(t *testing.T) {
// create poolmgr
port := 9999
err = StartExecutor(logger, functionNs, "fission-builder", port, true)
ctx := context.Background()
err = StartExecutor(ctx, logger, functionNs, "fission-builder", port, true)
if err != nil {
log.Panicf("failed to start poolmgr: %v", err)
}
pkg/executor/executortype/newdeploy/newdeploymgr.go
+2 -3
View File
@@ -141,7 +141,7 @@ func (deploy *NewDeploy) Run(ctx context.Context) {
if ok := k8sCache.WaitForCacheSync(ctx.Done(), deploy.deplListerSynced, deploy.svcListerSynced); !ok {
deploy.logger.Fatal("failed to wait for caches to sync")
}
go deploy.idleObjectReaper()
go deploy.idleObjectReaper(ctx)
}
// GetTypeName returns the executor type name.
@@ -748,8 +748,7 @@ func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message strin
}
// idleObjectReaper reaps objects after certain idle time
func (deploy *NewDeploy) idleObjectReaper() {
ctx := context.Background()
func (deploy *NewDeploy) idleObjectReaper(ctx context.Context) {
pollSleep := 5 * time.Second
for {
time.Sleep(pollSleep)
pkg/executor/reaper/reaper.go
+1 -2
View File
@@ -180,8 +180,7 @@ func CleanupHpa(ctx context.Context, logger *zap.Logger, client *kubernetes.Clie
// CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or
// deletes the rolebindings completely if there are no Service Accounts in a rolebinding object.
func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
ctx := context.Background()
func CleanupRoleBindings(ctx context.Context, logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
for {
// some sleep before the next reaper iteration
time.Sleep(cleanupRoleBindingInterval)
pkg/kubewatcher/kubewatcher.go
+19 -19
View File
@@ -77,7 +77,7 @@ type (
}
)
func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher {
func MakeKubeWatcher(ctx context.Context, logger *zap.Logger, kubernetesClient *kubernetes.Clientset, publisher publisher.Publisher) *KubeWatcher {
kw := &KubeWatcher{
logger: logger.Named("kube_watcher"),
watches: make(map[types.UID]watchSubscription),
@@ -85,7 +85,7 @@ func MakeKubeWatcher(logger *zap.Logger, kubernetesClient *kubernetes.Clientset,
publisher: publisher,
requestChannel: make(chan *kubeWatcherRequest),
}
go kw.svc()
go kw.svc(ctx)
return kw
}
@@ -100,7 +100,7 @@ func (kw *KubeWatcher) Sync(watches []fv1.KubernetesWatchTrigger) error {
return resp.error
}
func (kw *KubeWatcher) svc() {
func (kw *KubeWatcher) svc(ctx context.Context) {
for {
req := <-kw.requestChannel
switch req.requestType {
@@ -118,7 +118,7 @@ func (kw *KubeWatcher) svc() {
// Add new watches
for _, w := range req.watches {
if _, ok := kw.watches[w.ObjectMeta.UID]; !ok {
kw.addWatch(&w) //nolint: errCheck
kw.addWatch(ctx, &w) //nolint: errCheck
}
}
req.responseChannel <- &kubeWatcherResponse{error: nil}
@@ -149,7 +149,7 @@ func printKubernetesObject(obj runtime.Object, w io.Writer) error {
return err
}
func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) {
func createKubernetesWatch(ctx context.Context, kubeClient *kubernetes.Clientset, w *fv1.KubernetesWatchTrigger, resourceVersion string) (watch.Interface, error) {
var wi watch.Interface
var err error
var watchTimeoutSec int64 = 120
@@ -163,22 +163,22 @@ func createKubernetesWatch(kubeClient *kubernetes.Clientset, w *fv1.KubernetesWa
// TODO handle the full list of types
switch strings.ToUpper(w.Spec.Type) {
case "POD":
wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(context.TODO(), listOptions)
wi, err = kubeClient.CoreV1().Pods(w.Spec.Namespace).Watch(ctx, listOptions)
case "SERVICE":
wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(context.TODO(), listOptions)
wi, err = kubeClient.CoreV1().Services(w.Spec.Namespace).Watch(ctx, listOptions)
case "REPLICATIONCONTROLLER":
wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(context.TODO(), listOptions)
wi, err = kubeClient.CoreV1().ReplicationControllers(w.Spec.Namespace).Watch(ctx, listOptions)
case "JOB":
wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(context.TODO(), listOptions)
wi, err = kubeClient.BatchV1().Jobs(w.Spec.Namespace).Watch(ctx, listOptions)
default:
err = errors.NewBadRequest(fmt.Sprintf("Error: unknown obj type '%v'", w.Spec.Type))
}
return wi, err
}
func (kw *KubeWatcher) addWatch(w *fv1.KubernetesWatchTrigger) error {
func (kw *KubeWatcher) addWatch(ctx context.Context, w *fv1.KubernetesWatchTrigger) error {
kw.logger.Info("adding watch", zap.String("name", w.ObjectMeta.Name), zap.Any("function", w.Spec.FunctionReference))
ws, err := MakeWatchSubscription(kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher)
ws, err := MakeWatchSubscription(ctx, kw.logger.Named("watchsubscription"), w, kw.kubernetesClient, kw.publisher)
if err != nil {
return err
}
@@ -198,7 +198,7 @@ func (kw *KubeWatcher) removeWatch(w *fv1.KubernetesWatchTrigger) error {
return nil
}
func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) {
func MakeWatchSubscription(ctx context.Context, logger *zap.Logger, w *fv1.KubernetesWatchTrigger, kubeClient *kubernetes.Clientset, publisher publisher.Publisher) (*watchSubscription, error) {
var stopped int32 = 0
ws := &watchSubscription{
logger: logger.Named("watch_subscription"),
@@ -210,16 +210,16 @@ func MakeWatchSubscription(logger *zap.Logger, w *fv1.KubernetesWatchTrigger, ku
lastResourceVersion: "",
}
err := ws.restartWatch()
err := ws.restartWatch(ctx)
if err != nil {
return nil, err
}
go ws.eventDispatchLoop()
go ws.eventDispatchLoop(ctx)
return ws, nil
}
func (ws *watchSubscription) restartWatch() error {
func (ws *watchSubscription) restartWatch(ctx context.Context) error {
retries := 60
for {
ws.logger.Info("(re)starting watch",
@@ -227,7 +227,7 @@ func (ws *watchSubscription) restartWatch() error {
zap.String("namespace", ws.watch.Spec.Namespace),
zap.String("type", ws.watch.Spec.Type),
zap.String("last_resource_version", ws.lastResourceVersion))
wi, err := createKubernetesWatch(ws.kubernetesClient, &ws.watch, ws.lastResourceVersion)
wi, err := createKubernetesWatch(ctx, ws.kubernetesClient, &ws.watch, ws.lastResourceVersion)
if err != nil {
retries--
if retries > 0 {
@@ -250,7 +250,7 @@ func getResourceVersion(obj runtime.Object) (string, error) {
return m.GetResourceVersion(), nil
}
func (ws *watchSubscription) eventDispatchLoop() {
func (ws *watchSubscription) eventDispatchLoop(ctx context.Context) {
ws.logger.Info("listening to watch", zap.String("name", ws.watch.ObjectMeta.Name))
for {
// check watchSubscription is stopped or not before waiting for event
@@ -269,7 +269,7 @@ func (ws *watchSubscription) eventDispatchLoop() {
} else {
// watch closed due to timeout, restart it.
ws.logger.Warn("watch timed out - restarting", zap.String("watch_name", ws.watch.ObjectMeta.Name))
err := ws.restartWatch()
err := ws.restartWatch(ctx)
if err != nil {
ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name))
}
@@ -283,7 +283,7 @@ func (ws *watchSubscription) eventDispatchLoop() {
// Start from the beginning to get around "too old resource version"
ws.lastResourceVersion = ""
time.Sleep(time.Second)
err := ws.restartWatch()
err := ws.restartWatch(ctx)
if err != nil {
ws.logger.Panic("failed to restart watch", zap.Error(err), zap.String("watch_name", ws.watch.ObjectMeta.Name))
}
pkg/kubewatcher/main.go
+4 -2
View File
@@ -17,6 +17,8 @@ limitations under the License.
package kubewatcher
import (
"context"
"github.com/pkg/errors"
"go.uber.org/zap"
@@ -24,7 +26,7 @@ import (
"github.com/fission/fission/pkg/publisher"
)
func Start(logger *zap.Logger, routerUrl string) error {
func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error {
fissionClient, kubeClient, _, _, err := crd.MakeFissionClient()
if err != nil {
return errors.Wrap(err, "failed to get fission or kubernetes client")
@@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error {
}
poster := publisher.MakeWebhookPublisher(logger, routerUrl)
kubeWatch := MakeKubeWatcher(logger, kubeClient, poster)
kubeWatch := MakeKubeWatcher(ctx, logger, kubeClient, poster)
MakeWatchSync(logger, fissionClient, kubeWatch)
return nil
pkg/logger/logger.go
+4 -7
View File
@@ -17,6 +17,7 @@ limitations under the License.
package logger
import (
"context"
"fmt"
"log"
"os"
@@ -32,7 +33,6 @@ import (
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/crd"
"github.com/fission/fission/pkg/utils"
"github.com/fission/fission/pkg/utils/loggerfactory"
)
var nodeName = os.Getenv("NODE_NAME")
@@ -157,10 +157,7 @@ func symlinkReaper(zapLogger *zap.Logger) {
}
}
func Start() {
logger := loggerfactory.GetLogger()
defer logger.Sync()
func Start(ctx context.Context, logger *zap.Logger) {
if _, err := os.Stat(fissionSymlinkPath); os.IsNotExist(err) {
logger.Info("symlink path not exist, create it",
zap.String("fissionSymlinkPath", fissionSymlinkPath))
@@ -177,6 +174,6 @@ func Start() {
informerFactory := k8sInformers.NewSharedInformerFactory(kubernetesClient, time.Minute*30)
podInformer := informerFactory.Core().V1().Pods().Informer()
podInformer.AddEventHandler(podInformerHandlers(logger))
podInformer.Run(make(chan struct{}))
logger.Fatal("Stop watching pod changes")
podInformer.Run(ctx.Done())
logger.Error("Stop watching pod changes")
}
pkg/mqtrigger/scalermanager.go
+2 -2
View File
@@ -149,7 +149,7 @@ func mqTriggerEventHandlers(logger *zap.Logger, kubeClient *kubernetes.Clientset
// StartScalerManager watches for changes in MessageQueueTrigger and,
// Based on changes, it Creates, Updates and Deletes Objects of Kind ScaledObjects, AuthenticationTriggers and Deployments
func StartScalerManager(logger *zap.Logger, routerURL string) error {
func StartScalerManager(ctx context.Context, logger *zap.Logger, routerURL string) error {
fissionClient, kubeClient, _, _, err := crd.MakeFissionClient()
if err != nil {
return err
@@ -161,7 +161,7 @@ func StartScalerManager(logger *zap.Logger, routerURL string) error {
informerFactory := genInformer.NewSharedInformerFactory(fissionClient, time.Minute*30)
mqTriggerInformer := informerFactory.Core().V1().MessageQueueTriggers().Informer()
mqTriggerInformer.AddEventHandler(mqTriggerEventHandlers(logger, kubeClient, routerURL))
mqTriggerInformer.Run(context.Background().Done())
mqTriggerInformer.Run(ctx.Done())
return nil
}
pkg/router/router.go
+3 -5
View File
@@ -126,7 +126,7 @@ func serveMetric(logger *zap.Logger) {
}
// Start starts a router
func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) {
func Start(ctx context.Context, logger *zap.Logger, port int, executorURL string, openTracingEnabled bool) {
fmap := makeFunctionServiceMap(logger, time.Minute)
fissionClient, kubeClient, _, _, err := crd.MakeFissionClient()
@@ -258,10 +258,8 @@ func Start(logger *zap.Logger, port int, executorURL string, openTracingEnabled
logger.Info("starting router", zap.Int("port", port))
tracer := otel.Tracer("router")
ctx, span := tracer.Start(context.Background(), "router/Start")
ctx, span := tracer.Start(ctx, "router/Start")
defer span.End()
ctxWithCancel, cancel := context.WithCancel(ctx)
defer cancel()
serve(ctxWithCancel, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled)
serve(ctx, logger, port, tracingSamplingRate, triggers, displayAccessLog, openTracingEnabled)
}
pkg/storagesvc/archivePruner.go
+4 -4
View File
@@ -75,13 +75,13 @@ func (pruner *ArchivePruner) insertArchive(archiveID string) {
// A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes
// and not the archives that are referenced by them, leaving the archives as orphans.
// getOrphanArchives reaps the orphaned archives.
func (pruner *ArchivePruner) getOrphanArchives() {
func (pruner *ArchivePruner) getOrphanArchives(ctx context.Context) {
pruner.logger.Debug("getting orphan archives")
archivesRefByPkgs := make([]string, 0)
var archiveID string
// get all pkgs from kubernetes
pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
pkgList, err := pruner.crdClient.CoreV1().Packages(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
if err != nil {
pruner.logger.Error("error getting package list from kubernetes", zap.Error(err))
return
@@ -137,12 +137,12 @@ func (pruner *ArchivePruner) getOrphanArchives() {
// Start starts a go routine that listens to a channel for archive IDs that need to deleted.
// Also wakes up at regular intervals to make a list of archive IDs that need to be reaped
// and sends them over to the channel for deletion
func (pruner *ArchivePruner) Start() {
func (pruner *ArchivePruner) Start(ctx context.Context) {
ticker := time.NewTicker(pruner.pruneInterval * time.Minute)
go pruner.pruneArchives()
for range ticker.C {
// This method fetches unused archive IDs and sends them to archiveChannel for deletion
// silencing the errors, hoping they go away in next iteration.
pruner.getOrphanArchives()
pruner.getOrphanArchives(ctx)
}
}
pkg/storagesvc/client/storagesvc_test.go
+4 -4
View File
@@ -76,7 +76,7 @@ func runMinioDockerContainer(pool *dockertest.Pool) *dockertest.Resource {
return resource
}
func startS3StorageService(endpoint, bucketName, subDir string) {
func startS3StorageService(ctx context.Context, endpoint, bucketName, subDir string) {
// testID := uniuri.NewLen(8)
port := 8081
@@ -94,7 +94,7 @@ func startS3StorageService(endpoint, bucketName, subDir string) {
os.Setenv("STORAGE_S3_REGION", minioRegion)
storage := storagesvc.NewS3Storage()
_ = storagesvc.Start(logger, storage, port, true)
_ = storagesvc.Start(ctx, logger, storage, port, true)
}
func TestS3StorageService(t *testing.T) {
@@ -135,7 +135,7 @@ func TestS3StorageService(t *testing.T) {
// Start storagesvc
bucketName := "test-s3-service"
subDir := "x/y/z"
startS3StorageService(endpoint, bucketName, subDir)
startS3StorageService(context.Background(), endpoint, bucketName, subDir)
time.Sleep(time.Second)
client := MakeClient(fmt.Sprintf("http://localhost:%v/", 8081))
@@ -211,7 +211,7 @@ func TestLocalStorageService(t *testing.T) {
localPath := fmt.Sprintf("/tmp/%v", testID)
_ = os.Mkdir(localPath, os.ModePerm)
storage := storagesvc.NewLocalStorage(localPath)
_ = storagesvc.Start(logger, storage, port, true)
_ = storagesvc.Start(context.Background(), logger, storage, port, true)
time.Sleep(time.Second)
client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))
pkg/storagesvc/storagesvc.go
+3 -2
View File
@@ -17,6 +17,7 @@ limitations under the License.
package storagesvc
import (
"context"
"encoding/json"
"fmt"
"net/http"
@@ -221,7 +222,7 @@ func (ss *StorageService) Start(port int, openTracingEnabled bool) {
}
// Start runs storage service
func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error {
func Start(ctx context.Context, logger *zap.Logger, storage Storage, port int, openTracingEnabled bool) error {
enablePruner := true
// create a storage client
storageClient, err := MakeStowClient(logger, storage)
@@ -244,7 +245,7 @@ func Start(logger *zap.Logger, storage Storage, port int, openTracingEnabled boo
if err != nil {
return errors.Wrap(err, "Error creating archivePruner")
}
go pruner.Start()
go pruner.Start(ctx)
}
logger.Info("storage service started")
pkg/timer/main.go
+4 -2
View File
@@ -17,6 +17,8 @@ limitations under the License.
package timer
import (
"context"
"github.com/pkg/errors"
"go.uber.org/zap"
@@ -24,7 +26,7 @@ import (
"github.com/fission/fission/pkg/publisher"
)
func Start(logger *zap.Logger, routerUrl string) error {
func Start(ctx context.Context, logger *zap.Logger, routerUrl string) error {
fissionClient, _, _, _, err := crd.MakeFissionClient()
if err != nil {
return errors.Wrap(err, "failed to get fission or kubernetes client")
@@ -36,7 +38,7 @@ func Start(logger *zap.Logger, routerUrl string) error {
}
poster := publisher.MakeWebhookPublisher(logger, routerUrl)
MakeTimerSync(logger, fissionClient, MakeTimer(logger, poster))
MakeTimerSync(ctx, logger, fissionClient, MakeTimer(logger, poster))
return nil
}
pkg/timer/timerSync.go
+4 -4
View File
@@ -35,19 +35,19 @@ type (
}
)
func MakeTimerSync(logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync {
func MakeTimerSync(ctx context.Context, logger *zap.Logger, fissionClient *crd.FissionClient, timer *Timer) *TimerSync {
ws := &TimerSync{
logger: logger.Named("timer_sync"),
fissionClient: fissionClient,
timer: timer,
}
go ws.syncSvc()
go ws.syncSvc(ctx)
return ws
}
func (ws *TimerSync) syncSvc() {
func (ws *TimerSync) syncSvc(ctx context.Context) {
for {
triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
triggers, err := ws.fissionClient.CoreV1().TimeTriggers(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
if err != nil {
if utils.IsNetworkError(err) {
ws.logger.Info("encountered a network error - will retry", zap.Error(err))
pkg/tracker/tracker.go
+2 -2
View File
@@ -54,7 +54,7 @@ func init() {
Tracker = &tracker{gaPropertyID: os.Getenv(GA_TRACKING_ID), cid: id.String()}
}
func (t *tracker) SendEvent(e Event) error {
func (t *tracker) SendEvent(ctx context.Context, e Event) error {
if t.gaPropertyID == "" {
return errors.New("tracker.SendEvent: GA_TRACKING_ID env not set")
}
@@ -81,7 +81,7 @@ func (t *tracker) SendEvent(e Event) error {
}
buf := bytes.NewBufferString(v.Encode())
req, err := http.NewRequest("POST", GA_API_URL, buf)
req, err := http.NewRequestWithContext(ctx, "POST", GA_API_URL, buf)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req.Header.Add("User-Agent", "ga-tracker/1.0")
if err != nil {
pkg/utils/otel/provider.go
+2 -2
View File
@@ -216,11 +216,11 @@ func InitProvider(ctx context.Context, logger *zap.Logger, serviceName string) (
return func(ctx context.Context) {
err := tracerProvider.Shutdown(ctx)
if err != nil && logger != nil {
logger.Fatal("error shutting down trace provider", zap.Error(err))
logger.Error("error shutting down trace provider", zap.Error(err))
}
if traceExporter != nil {
if err = traceExporter.Shutdown(ctx); err != nil && logger != nil {
logger.Fatal("error shutting down trace exporter", zap.Error(err))
logger.Error("error shutting down trace exporter", zap.Error(err))
}
}
}, nil
pkg/utils/signals/signals.go
+46
View File
@@ -0,0 +1,46 @@
/*
Copyright 2021 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package signals
import (
"context"
"os"
"os/signal"
"syscall"
"go.uber.org/zap"
)
var onlyOneSignalHandler = make(chan struct{})
func SetupSignalHandlerWithContext(logger *zap.Logger) context.Context {
var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM}
close(onlyOneSignalHandler) // panics when called twice
ctx, cancel := context.WithCancel(context.Background())
c := make(chan os.Signal, 2)
signal.Notify(c, shutdownSignals...)
go func() {
signal := <-c
logger.Info("Received signal", zap.String("signal", signal.String()))
cancel()
<-c
panic("multiple signals received")
}()
return ctx
}