Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Websocket Support: Pod Reaper (#1982)

Browse Source 
Websocket event support for cleaning up pods only after WS connection is terminated. The support for websocket is right now in the environment itself and router simply acts as a proxy for WS communication!
This commit is contained in:
Harsh Thakur
2021-04-27 21:06:22 +05:30
committed by GitHub
parent 0d93c109fb
commit f72bfd8f3a
15 changed files with 294 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/core/v1/const.go
+2
View File
@@ -66,6 +66,8 @@ const (
SharedVolumePackages = "packages"
SharedVolumeSecrets = "secrets"
SharedVolumeConfigmaps = "configmaps"
PodInfoVolume = "podinfo"
PodInfoMount = "/etc/podinfo"
)
const (
pkg/executor/executortype/poolmgr/gp.go
+7 -1
View File
@@ -77,7 +77,8 @@ type (
readyPodQueue workqueue.DelayingInterface
poolInstanceID string // small random string to uniquify pod names
instanceID string // poolmgr instance id
podFSVCMap sync.Map
// TODO: move this field into fsCache
podFSVCMap sync.Map
}
)
@@ -688,6 +689,11 @@ func (gp *GenericPool) getFuncSvc(ctx context.Context, fn *fv1.Function) (*fscac
Atime: time.Now(),
}
if gp.fsCache.PodToFsvc == nil {
gp.fsCache.PodToFsvc = make(map[string]*fscache.FuncSvc)
}
gp.fsCache.PodToFsvc[pod.GetObjectMeta().GetName()] = fsvc
gp.podFSVCMap.Store(pod.ObjectMeta.Name, []interface{}{crd.CacheKey(fsvc.Function), fsvc.Address})
gp.fsCache.AddFunc(*fsvc)
pkg/executor/executortype/poolmgr/gpm.go
+95
View File
@@ -32,10 +32,13 @@ import (
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
k8sTypes "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
k8sInformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
k8sCache "k8s.io/client-go/tools/cache"
k8scache "k8s.io/client-go/tools/cache"
metricsclient "k8s.io/metrics/pkg/client/clientset/versioned"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
@@ -598,6 +601,11 @@ func (gpm *GenericPoolManager) getEnvPoolsize(env *fv1.Environment) int32 {
func (gpm *GenericPoolManager) idleObjectReaper() {
pollSleep := 5 * time.Second
go gpm.WebsocketStartEventChecker(gpm.kubernetesClient)
go gpm.NoActiveConnectionEventChecker(gpm.kubernetesClient)
for {
time.Sleep(pollSleep)
@@ -636,6 +644,9 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
continue
}
if _, ok := gpm.fsCache.WebsocketFsvc[fsvc.Name]; ok {
continue
}
// For function with the environment that no longer exists, executor
// cleanups the idle pod as usual and prints log to notify user.
if _, ok := envList[fsvc.Environment.ObjectMeta.UID]; !ok {
@@ -682,3 +693,87 @@ func (gpm *GenericPoolManager) idleObjectReaper() {
}
}
}
// WebsocketStartEventChecker checks if the pod has emitted a websocket connection start event
func (gpm *GenericPoolManager) WebsocketStartEventChecker(kubeClient *kubernetes.Clientset) {
informer := k8scache.NewSharedInformer(
&k8scache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = "involvedObject.kind=Pod,type=Normal,reason=WsConnectionStarted"
return kubeClient.CoreV1().Events(apiv1.NamespaceAll).List(options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = "involvedObject.kind=Pod,type=Normal,reason=WsConnectionStarted"
return kubeClient.CoreV1().Events(apiv1.NamespaceAll).Watch(options)
},
},
&apiv1.Event{},
0,
)
stopper := make(chan struct{})
defer close(stopper)
informer.AddEventHandler(k8scache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
mObj := obj.(metav1.Object)
gpm.logger.Info("Websocket event detected for pod",
zap.String("Pod name", mObj.GetName()))
podName := strings.SplitAfter(mObj.GetName(), ".")
if fsvc, ok := gpm.fsCache.PodToFsvc[strings.TrimSuffix(podName[0], ".")]; ok {
gpm.fsCache.WebsocketFsvc[fsvc.Name] = true
}
},
})
informer.Run(stopper)
}
// NoActiveConnectionEventChecker checks if the pod has emitted an inactive event
func (gpm *GenericPoolManager) NoActiveConnectionEventChecker(kubeClient *kubernetes.Clientset) {
informer := k8scache.NewSharedInformer(
&k8scache.ListWatch{
ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = "involvedObject.kind=Pod,type=Normal,reason=NoActiveConnections"
return kubeClient.CoreV1().Events(apiv1.NamespaceAll).List(options)
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = "involvedObject.kind=Pod,type=Normal,reason=NoActiveConnections"
return kubeClient.CoreV1().Events(apiv1.NamespaceAll).Watch(options)
},
},
&apiv1.Event{},
0,
)
stopper := make(chan struct{})
defer close(stopper)
informer.AddEventHandler(k8scache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
mObj := obj.(metav1.Object)
gpm.logger.Info("Inactive event detected for pod",
zap.String("Pod name", mObj.GetName()))
podName := strings.SplitAfter(mObj.GetName(), ".")
if fsvc, ok := gpm.fsCache.PodToFsvc[strings.TrimSuffix(podName[0], ".")]; ok {
gpm.fsCache.DeleteFunctionSvc(fsvc)
for i := range fsvc.KubernetesObjects {
gpm.logger.Info("release idle function resources due to inactivity",
zap.String("function", fsvc.Function.Name),
zap.String("address", fsvc.Address),
zap.String("executor", string(fsvc.Executor)),
zap.String("pod", fsvc.Name),
)
reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &fsvc.KubernetesObjects[i])
time.Sleep(50 * time.Millisecond)
}
}
},
})
informer.Run(stopper)
}
pkg/executor/fscache/functionServiceCache.go
+5 -2
View File
@@ -68,8 +68,9 @@ type (
byAddress *cache.Cache // address -> function : map[string]metav1.ObjectMeta
byFunctionUID *cache.Cache // function uid -> function : map[string]metav1.ObjectMeta
connFunctionCache *poolcache.Cache // function-key -> funcSvc : map[string]*funcSvc
requestChannel chan *fscRequest
PodToFsvc map[string]*FuncSvc
WebsocketFsvc map[string]bool
requestChannel chan *fscRequest
}
fscRequest struct {
@@ -110,6 +111,8 @@ func MakeFunctionServiceCache(logger *zap.Logger) *FunctionServiceCache {
byFunctionUID: cache.MakeCache(0, 0),
connFunctionCache: poolcache.NewPoolCache(),
requestChannel: make(chan *fscRequest),
PodToFsvc: make(map[string]*FuncSvc),
WebsocketFsvc: make(map[string]bool),
}
go fsc.service()
return fsc
pkg/fetcher/config/config.go
+31
View File
@@ -176,6 +176,27 @@ func (cfg *Config) fetcherCommand(extraArgs ...string) []string {
}
func (cfg *Config) volumesWithMounts() ([]apiv1.Volume, []apiv1.VolumeMount) {
items := make([]apiv1.DownwardAPIVolumeFile, 0)
podNameFieldSelector := apiv1.ObjectFieldSelector{
FieldPath: "metadata.name",
}
podNamespaceFieldSelector := apiv1.ObjectFieldSelector{
FieldPath: "metadata.namespace",
}
podName := apiv1.DownwardAPIVolumeFile{
Path: "name",
FieldRef: &podNameFieldSelector,
}
podNamespace := apiv1.DownwardAPIVolumeFile{
Path: "namespace",
FieldRef: &podNamespaceFieldSelector,
}
items = append(items, podName, podNamespace)
dwAPIVol := apiv1.DownwardAPIVolumeSource{Items: items}
volumes := []apiv1.Volume{
{
Name: fv1.SharedVolumeUserfunc,
@@ -195,6 +216,12 @@ func (cfg *Config) volumesWithMounts() ([]apiv1.Volume, []apiv1.VolumeMount) {
EmptyDir: &apiv1.EmptyDirVolumeSource{},
},
},
{
Name: fv1.PodInfoVolume,
VolumeSource: apiv1.VolumeSource{
DownwardAPI: &dwAPIVol,
},
},
}
mounts := []apiv1.VolumeMount{
{
@@ -209,6 +236,10 @@ func (cfg *Config) volumesWithMounts() ([]apiv1.Volume, []apiv1.VolumeMount) {
Name: fv1.SharedVolumeConfigmaps,
MountPath: cfg.sharedCfgMapPath,
},
{
Name: fv1.PodInfoVolume,
MountPath: fv1.PodInfoMount,
},
}
return volumes, mounts
pkg/fetcher/fetcher.go
+100
View File
@@ -32,9 +32,15 @@ import (
uuid "github.com/satori/go.uuid"
"go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
k8serr "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/tools/reference"
"k8s.io/klog"
fv1 "github.com/fission/fission/pkg/apis/core/v1"
"github.com/fission/fission/pkg/crd"
@@ -54,6 +60,11 @@ type (
fissionClient *crd.FissionClient
kubeClient *kubernetes.Clientset
httpClient *http.Client
Info PodInfo
}
PodInfo struct {
Name string
Namespace string
}
)
@@ -80,6 +91,17 @@ func MakeFetcher(logger *zap.Logger, sharedVolumePath string, sharedSecretPath s
if err != nil {
return nil, errors.Wrap(err, "error making the fission / kube client")
}
name, err := ioutil.ReadFile(fv1.PodInfoMount + "/name")
if err != nil {
return nil, errors.Wrap(err, "error reading pod name from downward volume")
}
namespace, err := ioutil.ReadFile(fv1.PodInfoMount + "/namespace")
if err != nil {
return nil, errors.Wrap(err, "error reading pod namespace from downward volume")
}
return &Fetcher{
logger: fLogger,
sharedVolumePath: sharedVolumePath,
@@ -87,6 +109,10 @@ func MakeFetcher(logger *zap.Logger, sharedVolumePath string, sharedSecretPath s
sharedConfigPath: sharedConfigPath,
fissionClient: fissionClient,
kubeClient: kubeClient,
Info: PodInfo{
Name: string(name),
Namespace: string(namespace),
},
httpClient: &http.Client{
Transport: &ochttp.Transport{},
},
@@ -662,3 +688,77 @@ func (fetcher *Fetcher) SpecializePod(ctx context.Context, fetchReq FunctionFetc
return errors.Wrapf(err, "error specializing function pod after %v times", maxRetries)
}
// WsStartHandler is used to generate websocket events in Kubernetes
func (fetcher *Fetcher) WsStartHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "only GET is supported on this endpoint", http.StatusMethodNotAllowed)
return
}
rec, err := eventRecorder(fetcher.kubeClient)
if err != nil {
klog.Errorf("Error creating recorder %s", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
}
pods, err := fetcher.kubeClient.CoreV1().Pods(fetcher.Info.Namespace).List(metav1.ListOptions{
FieldSelector: "metadata.name=" + fetcher.Info.Name,
})
if err != nil {
fetcher.logger.Error("Failed to get the pod", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError)
}
for _, pod := range pods.Items {
ref, err := reference.GetReference(scheme.Scheme, &pod)
if err != nil {
fetcher.logger.Error("Could not get reference for pod", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError)
}
rec.Event(ref, corev1.EventTypeNormal, "WsConnectionStarted", "Websocket connection has been formed on this pod")
fetcher.logger.Info("Sent websocket initiation event")
}
w.WriteHeader(http.StatusOK)
}
// WsEndHandler is used to generate inactive events in Kubernetes
func (fetcher *Fetcher) WsEndHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "only GET is supported on this endpoint", http.StatusMethodNotAllowed)
return
}
rec, err := eventRecorder(fetcher.kubeClient)
if err != nil {
klog.Errorf("Error creating recorder %s", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
}
pods, err := fetcher.kubeClient.CoreV1().Pods(fetcher.Info.Namespace).List(metav1.ListOptions{
FieldSelector: "metadata.name=" + fetcher.Info.Name,
})
if err != nil {
fetcher.logger.Error("Failed to get the pod", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError)
}
for _, pod := range pods.Items {
// There will only be one time since we've used field selector
ref, err := reference.GetReference(scheme.Scheme, &pod)
if err != nil {
fetcher.logger.Error("Could not get reference for pod", zap.Error(err))
http.Error(w, err.Error(), http.StatusInternalServerError)
}
// We could use Eventf and supply the amount of time the connection was inactive although, in case of multiple connections, it doesn't make sense
rec.Event(ref, corev1.EventTypeNormal, "NoActiveConnections", "Connection has been inactive")
fetcher.logger.Info("Sent no active connections event")
}
w.WriteHeader(http.StatusOK)
}
func eventRecorder(kubeClient *kubernetes.Clientset) (record.EventRecorder, error) {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(zap.S().Infof)
eventBroadcaster.StartRecordingToSink(
&typedcorev1.EventSinkImpl{
Interface: kubeClient.CoreV1().Events("")})
recorder := eventBroadcaster.NewRecorder(
scheme.Scheme,
corev1.EventSource{Component: "fetcher"})
return recorder, nil
}