fission-src/pkg/buildermgr/envwatcher.go

/*
Copyright 2017 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package buildermgr

import (
	"context"
	"fmt"
	"os"
	"strconv"
	"time"

	"github.com/pkg/errors"
	"go.uber.org/zap"
	appsv1 "k8s.io/api/apps/v1"
	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/kubernetes"
	k8sCache "k8s.io/client-go/tools/cache"

	fv1 "github.com/fission/fission/pkg/apis/core/v1"
	"github.com/fission/fission/pkg/crd"
	"github.com/fission/fission/pkg/executor/util"
	fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
	"github.com/fission/fission/pkg/generated/clientset/versioned"
	"github.com/fission/fission/pkg/utils"
)

const (
	LABEL_ENV_NAME            = "envName"
	LABEL_ENV_NAMESPACE       = "envNamespace"
	LABEL_ENV_RESOURCEVERSION = "envResourceVersion"
	LABEL_DEPLOYMENT_OWNER    = "owner"
	BUILDER_MGR               = "buildermgr"
)

var (
	deletePropagation = metav1.DeletePropagationBackground
	delOpt            = metav1.DeleteOptions{PropagationPolicy: &deletePropagation}
)

type (
	builderInfo struct {
		envMetadata *metav1.ObjectMeta
		deployment  *appsv1.Deployment
		service     *apiv1.Service
	}

	environmentWatcher struct {
		logger                 *zap.Logger
		cache                  map[string]*builderInfo
		fissionClient          versioned.Interface
		kubernetesClient       kubernetes.Interface
		nsResolver             *utils.NamespaceResolver
		fetcherConfig          *fetcherConfig.Config
		builderImagePullPolicy apiv1.PullPolicy
		useIstio               bool
		podSpecPatch           *apiv1.PodSpec
		envWatchInformer       map[string]k8sCache.SharedIndexInformer
	}
)

func makeEnvironmentWatcher(
	ctx context.Context,
	logger *zap.Logger,
	fissionClient versioned.Interface,
	kubernetesClient kubernetes.Interface,
	fetcherConfig *fetcherConfig.Config,
	podSpecPatch *apiv1.PodSpec) *environmentWatcher {

	useIstio := false
	enableIstio := os.Getenv("ENABLE_ISTIO")
	if len(enableIstio) > 0 {
		istio, err := strconv.ParseBool(enableIstio)
		if err != nil {
			logger.Error("Failed to parse ENABLE_ISTIO, defaults to false")
		}
		useIstio = istio
	}

	builderImagePullPolicy := utils.GetImagePullPolicy(os.Getenv("BUILDER_IMAGE_PULL_POLICY"))

	envWatcher := &environmentWatcher{
		logger:                 logger.Named("environment_watcher"),
		cache:                  make(map[string]*builderInfo),
		fissionClient:          fissionClient,
		kubernetesClient:       kubernetesClient,
		nsResolver:             utils.DefaultNSResolver(),
		builderImagePullPolicy: builderImagePullPolicy,
		useIstio:               useIstio,
		fetcherConfig:          fetcherConfig,
		podSpecPatch:           podSpecPatch,
		envWatchInformer:       utils.GetInformersForNamespaces(fissionClient, time.Minute*30, fv1.EnvironmentResource),
	}

	envWatcher.EnvWatchEventHandlers(ctx)
	return envWatcher
}

func (env *environmentWatcher) getDeploymentLabels(envName string) map[string]string {
	return map[string]string{
		LABEL_DEPLOYMENT_OWNER: BUILDER_MGR,
		LABEL_ENV_NAME:         envName,
	}
}

func (envw *environmentWatcher) getLabels(envName string, envNamespace string, envResourceVersion string) map[string]string {
	return map[string]string{
		LABEL_ENV_NAME:            envName,
		LABEL_ENV_NAMESPACE:       envNamespace,
		LABEL_ENV_RESOURCEVERSION: envResourceVersion,
		LABEL_DEPLOYMENT_OWNER:    BUILDER_MGR,
	}
}

func (envw *environmentWatcher) Run(ctx context.Context) {
	for _, informer := range envw.envWatchInformer {
		go informer.Run(ctx.Done())
	}
}

func (envw *environmentWatcher) EnvWatchEventHandlers(ctx context.Context) {
	for _, informer := range envw.envWatchInformer {
		informer.AddEventHandler(k8sCache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				envObj := obj.(*fv1.Environment)
				envw.AddUpdateBuilder(ctx, envObj)
			},
			UpdateFunc: func(oldObj interface{}, newObj interface{}) {
				oldEnvObj := oldObj.(*fv1.Environment)
				newEnvObj := newObj.(*fv1.Environment)
				if oldEnvObj.ObjectMeta.ResourceVersion != newEnvObj.ObjectMeta.ResourceVersion {
					envw.AddUpdateBuilder(ctx, newEnvObj)
				}
			},
			DeleteFunc: func(obj interface{}) {
				envObj := obj.(*fv1.Environment)
				envw.DeleteBuilder(ctx, envObj)
			},
		})
	}
}

func (envw *environmentWatcher) AddUpdateBuilder(ctx context.Context, env *fv1.Environment) {
	//builder is not supported with v1 interface and ignore env without builder image
	if env.Spec.Version != 1 && len(env.Spec.Builder.Image) != 0 {
		if _, ok := envw.cache[crd.CacheKeyUID(&env.ObjectMeta)]; !ok {
			builderInfo, err := envw.createBuilder(ctx, env, envw.nsResolver.GetBuilderNS(env.ObjectMeta.Namespace))
			if err != nil {
				envw.logger.Error("error creating builder service", zap.Error(err))
				return
			}
			envw.cache[crd.CacheKeyUID(&env.ObjectMeta)] = builderInfo
		} else {
			envw.DeleteBuilder(ctx, env)
			// once older builder deleted then add new builder service
			builderInfo, err := envw.createBuilder(ctx, env, envw.nsResolver.GetBuilderNS(env.ObjectMeta.Namespace))
			if err != nil {
				envw.logger.Error("error updating builder service", zap.Error(err))
				return
			}
			envw.cache[crd.CacheKeyUID(&env.ObjectMeta)] = builderInfo
		}
	}
}

func (envw *environmentWatcher) DeleteBuilder(ctx context.Context, env *fv1.Environment) {
	if _, ok := envw.cache[crd.CacheKeyUID(&env.ObjectMeta)]; ok {
		envw.DeleteBuilderService(ctx, env)
		envw.DeleteBuilderDeployment(ctx, env)
		delete(envw.cache, crd.CacheKeyUID(&env.ObjectMeta))
		envw.logger.Info("builder service deleted", zap.String("env_name", env.ObjectMeta.Name), zap.String("namespace", envw.nsResolver.GetBuilderNS(env.ObjectMeta.Namespace)))
	} else {
		envw.logger.Debug("builder service not found", zap.String("env_name", env.ObjectMeta.Name), zap.String("namespace", envw.nsResolver.GetBuilderNS(env.ObjectMeta.Namespace)))
	}
}

func (envw *environmentWatcher) DeleteBuilderService(ctx context.Context, env *fv1.Environment) {
	ns := envw.nsResolver.GetBuilderNS(env.ObjectMeta.Namespace)
	svcList, err := envw.getBuilderServiceList(ctx, envw.getDeploymentLabels(env.ObjectMeta.Name), ns)
	if err != nil {
		envw.logger.Error("error getting the builder service list", zap.Error(err))
	}
	for _, svc := range svcList {
		envName := svc.ObjectMeta.Labels[LABEL_ENV_NAME]
		if _, ok := envw.cache[crd.CacheKeyUID(&env.ObjectMeta)]; ok {
			err := envw.deleteBuilderServiceByName(ctx, svc.ObjectMeta.Name, svc.ObjectMeta.Namespace)
			if err != nil {
				envw.logger.Error("error removing builder service", zap.Error(err),
					zap.String("service_name", svc.ObjectMeta.Name),
					zap.String("service_namespace", svc.ObjectMeta.Namespace),
					zap.String("env_name", envName))
			}
			break
		} else {
			envw.logger.Error("builder service not found",
				zap.String("service_name", svc.ObjectMeta.Name),
				zap.String("service_namespace", svc.ObjectMeta.Namespace))
		}
	}
}

func (envw *environmentWatcher) DeleteBuilderDeployment(ctx context.Context, env *fv1.Environment) {
	ns := envw.nsResolver.GetBuilderNS(env.ObjectMeta.Namespace)
	deployList, err := envw.getBuilderDeploymentList(ctx, envw.getDeploymentLabels(env.ObjectMeta.Name), ns)
	if err != nil {
		envw.logger.Error("error getting the builder deployment list", zap.Error(err))
	}
	for _, deploy := range deployList {
		if _, ok := envw.cache[crd.CacheKeyUID(&env.ObjectMeta)]; ok {
			err := envw.deleteBuilderDeploymentByName(ctx, deploy.ObjectMeta.Name, deploy.ObjectMeta.Namespace)
			if err != nil {
				envw.logger.Error("error removing builder deployment", zap.Error(err),
					zap.String("deployment_name", deploy.ObjectMeta.Name),
					zap.String("deployment_namespace", deploy.ObjectMeta.Namespace))
			}
			break
		} else {
			envw.logger.Error("builder deployment not found", zap.Error(err),
				zap.String("deployment_name", deploy.ObjectMeta.Name),
				zap.String("deployment_namespace", deploy.ObjectMeta.Namespace))
		}
	}
}

func (envw *environmentWatcher) createBuilder(ctx context.Context, env *fv1.Environment, ns string) (*builderInfo, error) {
	var svc *apiv1.Service
	var deploy *appsv1.Deployment

	sel := envw.getLabels(env.ObjectMeta.Name, ns, env.ObjectMeta.ResourceVersion)

	svcList, err := envw.getBuilderServiceList(ctx, sel, ns)
	if err != nil {
		return nil, err
	}
	// there should be only one service in svcList
	if len(svcList) == 0 {
		svc, err = envw.createBuilderService(ctx, env, ns)
		if err != nil {
			return nil, errors.Wrap(err,
				fmt.Sprintf("error creating builder service for environment in namespace %s %s", env.ObjectMeta.Name, ns))

		}
	} else if len(svcList) == 1 {
		svc = &svcList[0]
	} else {
		return nil, fmt.Errorf("found more than one builder service for environment in namespace %s %s", env.ObjectMeta.Name, ns)
	}

	deployList, err := envw.getBuilderDeploymentList(ctx, sel, ns)
	if err != nil {
		return nil, err
	}
	// there should be only one deploy in deployList
	if len(deployList) == 0 {
		// create builder SA in this ns, if not already created
		_, err := utils.SetupSA(ctx, envw.kubernetesClient, fv1.FissionBuilderSA, ns)
		if err != nil {
			return nil, errors.Wrapf(err, "error creating %q in ns: %s", fv1.FissionBuilderSA, ns)
		}

		deploy, err = envw.createBuilderDeployment(ctx, env, ns)
		if err != nil {
			return nil, errors.Wrap(err, fmt.Sprintf("error creating builder deployment for environment in namespace %s %s", env.ObjectMeta.Name, ns))

		}
	} else if len(deployList) == 1 {
		deploy = &deployList[0]
	} else {
		return nil, fmt.Errorf("found more than one builder deployment for environment in namespace %s %s", env.ObjectMeta.Name, ns)
	}

	return &builderInfo{
		envMetadata: &env.ObjectMeta,
		service:     svc,
		deployment:  deploy,
	}, nil
}

func (envw *environmentWatcher) deleteBuilderServiceByName(ctx context.Context, name, namespace string) error {
	err := envw.kubernetesClient.CoreV1().
		Services(namespace).
		Delete(ctx, name, delOpt)
	if err != nil {
		return errors.Wrapf(err, "error deleting builder service %s.%s", name, namespace)
	}
	return nil
}

func (envw *environmentWatcher) deleteBuilderDeploymentByName(ctx context.Context, name, namespace string) error {
	err := envw.kubernetesClient.AppsV1().
		Deployments(namespace).
		Delete(ctx, name, delOpt)
	if err != nil {
		return errors.Wrapf(err, "error deleting builder deployment %s.%s", name, namespace)
	}
	return nil
}

func (envw *environmentWatcher) getBuilderServiceList(ctx context.Context, sel map[string]string, ns string) ([]apiv1.Service, error) {
	svcList, err := envw.kubernetesClient.CoreV1().Services(ns).List(
		ctx,
		metav1.ListOptions{
			LabelSelector: labels.Set(sel).AsSelector().String(),
		})
	if err != nil {
		return nil, errors.Wrap(err, "error getting builder service list")
	}
	return svcList.Items, nil
}

func (envw *environmentWatcher) createBuilderService(ctx context.Context, env *fv1.Environment, ns string) (*apiv1.Service, error) {
	name := fmt.Sprintf("%v-%v", env.ObjectMeta.Name, env.ObjectMeta.ResourceVersion)
	sel := envw.getLabels(env.ObjectMeta.Name, ns, env.ObjectMeta.ResourceVersion)
	service := apiv1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: ns,
			Name:      name,
			Labels:    sel,
		},
		Spec: apiv1.ServiceSpec{
			Selector: sel,
			Type:     apiv1.ServiceTypeClusterIP,
			Ports: []apiv1.ServicePort{
				{
					Name:     "fetcher-port",
					Protocol: apiv1.ProtocolTCP,
					Port:     8000,
					TargetPort: intstr.IntOrString{
						Type:   intstr.Int,
						IntVal: 8000,
					},
				},
				{
					Name:     "builder-port",
					Protocol: apiv1.ProtocolTCP,
					Port:     8001,
					TargetPort: intstr.IntOrString{
						Type:   intstr.Int,
						IntVal: 8001,
					},
				},
			},
		},
	}
	envw.logger.Info("creating builder service", zap.String("service_name", name))
	_, err := envw.kubernetesClient.CoreV1().Services(ns).Create(ctx, &service, metav1.CreateOptions{})
	if err != nil {
		return nil, err
	}
	return &service, nil
}

func (envw *environmentWatcher) getBuilderDeploymentList(ctx context.Context, sel map[string]string, ns string) ([]appsv1.Deployment, error) {
	deployList, err := envw.kubernetesClient.AppsV1().Deployments(ns).List(
		ctx,
		metav1.ListOptions{
			LabelSelector: labels.Set(sel).AsSelector().String(),
		})
	if err != nil {
		return nil, errors.Wrap(err, "error getting builder deployment list")
	}
	return deployList.Items, nil
}

func (envw *environmentWatcher) createBuilderDeployment(ctx context.Context, env *fv1.Environment, ns string) (*appsv1.Deployment, error) {
	name := fmt.Sprintf("%v-%v", env.ObjectMeta.Name, env.ObjectMeta.ResourceVersion)
	sel := envw.getLabels(env.ObjectMeta.Name, ns, env.ObjectMeta.ResourceVersion)
	var replicas int32 = 1

	podAnnotations := env.ObjectMeta.Annotations
	if podAnnotations == nil {
		podAnnotations = make(map[string]string)
	}
	if envw.useIstio && env.Spec.AllowAccessToExternalNetwork {
		podAnnotations["sidecar.istio.io/inject"] = "false"
	}

	container, err := util.MergeContainer(&apiv1.Container{
		Name:                   "builder",
		Image:                  env.Spec.Builder.Image,
		ImagePullPolicy:        envw.builderImagePullPolicy,
		TerminationMessagePath: "/dev/termination-log",
		Command:                []string{"/builder", envw.fetcherConfig.SharedMountPath()},
		ReadinessProbe: &apiv1.Probe{
			InitialDelaySeconds: 5,
			PeriodSeconds:       2,
			ProbeHandler: apiv1.ProbeHandler{
				HTTPGet: &apiv1.HTTPGetAction{
					Path: "/healthz",
					Port: intstr.IntOrString{
						Type:   intstr.Int,
						IntVal: 8001,
					},
				},
			},
		},
	}, env.Spec.Builder.Container)
	if err != nil {
		return nil, err
	}

	pod := apiv1.PodTemplateSpec{
		ObjectMeta: metav1.ObjectMeta{
			Labels:      sel,
			Annotations: podAnnotations,
		},
		Spec: apiv1.PodSpec{
			Containers:         []apiv1.Container{*container},
			ServiceAccountName: fv1.FissionBuilderSA,
		},
	}

	if envw.podSpecPatch != nil {

		updatedPodSpec, err := util.MergePodSpec(&pod.Spec, envw.podSpecPatch)
		if err == nil {
			pod.Spec = *updatedPodSpec
		} else {
			envw.logger.Warn("Failed to merge the specs: %v", zap.Error(err))
		}
	}

	pod.Spec = *(util.ApplyImagePullSecret(env.Spec.ImagePullSecret, pod.Spec))

	deployment := &appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: ns,
			Name:      name,
			Labels:    sel,
		},
		Spec: appsv1.DeploymentSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{
				MatchLabels: sel,
			},
			Template: pod,
		},
	}

	err = envw.fetcherConfig.AddFetcherToPodSpec(&deployment.Spec.Template.Spec, "builder")
	if err != nil {
		return nil, err
	}

	if env.Spec.Builder.PodSpec != nil {
		newPodSpec, err := util.MergePodSpec(&deployment.Spec.Template.Spec, env.Spec.Builder.PodSpec)
		if err != nil {
			return nil, err
		}
		deployment.Spec.Template.Spec = *newPodSpec
	}

	_, err = envw.kubernetesClient.AppsV1().Deployments(ns).Create(ctx, deployment, metav1.CreateOptions{})
	if err != nil {
		return nil, err
	}

	envw.logger.Info("creating builder deployment", zap.String("deployment", name))

	return deployment, nil
}