Ta-Ching ChenandGitHub
b4300feabc
Newdeploy manager now checks the existence of function service cache by function UID before trying to create a new deployment. And return the cached fsvc directly if the cache exists.
556 lines
16 KiB
Go
556 lines
16 KiB
Go
/*
|
|
Copyright 2016 The Fission Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package newdeploy
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/fields"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/pkg/api"
|
|
apiv1 "k8s.io/client-go/pkg/api/v1"
|
|
"k8s.io/client-go/rest"
|
|
k8sCache "k8s.io/client-go/tools/cache"
|
|
|
|
"github.com/fission/fission"
|
|
"github.com/fission/fission/crd"
|
|
"github.com/fission/fission/executor/fscache"
|
|
)
|
|
|
|
type (
|
|
requestType int
|
|
|
|
NewDeploy struct {
|
|
kubernetesClient *kubernetes.Clientset
|
|
fissionClient *crd.FissionClient
|
|
crdClient *rest.RESTClient
|
|
instanceID string
|
|
|
|
fetcherImg string
|
|
fetcherImagePullPolicy apiv1.PullPolicy
|
|
namespace string
|
|
sharedMountPath string
|
|
sharedSecretPath string
|
|
sharedCfgMapPath string
|
|
useIstio bool
|
|
|
|
fsCache *fscache.FunctionServiceCache // cache funcSvc's by function, address and podname
|
|
requestChannel chan *fnRequest
|
|
|
|
functions []crd.Function
|
|
funcStore k8sCache.Store
|
|
funcController k8sCache.Controller
|
|
}
|
|
|
|
fnRequest struct {
|
|
reqType requestType
|
|
fn *crd.Function
|
|
responseChannel chan *fnResponse
|
|
}
|
|
|
|
fnResponse struct {
|
|
error
|
|
fSvc *fscache.FuncSvc
|
|
}
|
|
)
|
|
|
|
const (
|
|
FnCreate requestType = iota
|
|
FnUpdate
|
|
FnDelete
|
|
)
|
|
|
|
func MakeNewDeploy(
|
|
fissionClient *crd.FissionClient,
|
|
kubernetesClient *kubernetes.Clientset,
|
|
crdClient *rest.RESTClient,
|
|
namespace string,
|
|
fsCache *fscache.FunctionServiceCache,
|
|
instanceID string,
|
|
) *NewDeploy {
|
|
|
|
log.Printf("Creating NewDeploy ExecutorType")
|
|
|
|
fetcherImg := os.Getenv("FETCHER_IMAGE")
|
|
if len(fetcherImg) == 0 {
|
|
fetcherImg = "fission/fetcher"
|
|
}
|
|
fetcherImagePullPolicy := os.Getenv("FETCHER_IMAGE_PULL_POLICY")
|
|
if len(fetcherImagePullPolicy) == 0 {
|
|
fetcherImagePullPolicy = "IfNotPresent"
|
|
}
|
|
|
|
enableIstio := false
|
|
if len(os.Getenv("ENABLE_ISTIO")) > 0 {
|
|
istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
|
|
if err != nil {
|
|
log.Println("Failed to parse ENABLE_ISTIO")
|
|
}
|
|
enableIstio = istio
|
|
}
|
|
|
|
nd := &NewDeploy{
|
|
fissionClient: fissionClient,
|
|
kubernetesClient: kubernetesClient,
|
|
crdClient: crdClient,
|
|
instanceID: instanceID,
|
|
|
|
namespace: namespace,
|
|
fsCache: fsCache,
|
|
|
|
fetcherImg: fetcherImg,
|
|
fetcherImagePullPolicy: apiv1.PullIfNotPresent,
|
|
sharedMountPath: "/userfunc",
|
|
sharedSecretPath: "/secrets",
|
|
sharedCfgMapPath: "/configs",
|
|
useIstio: enableIstio,
|
|
|
|
requestChannel: make(chan *fnRequest),
|
|
}
|
|
|
|
if nd.crdClient != nil {
|
|
fnStore, fnController := nd.initFuncController()
|
|
nd.funcStore = fnStore
|
|
nd.funcController = fnController
|
|
}
|
|
go nd.service()
|
|
return nd
|
|
}
|
|
|
|
func (deploy *NewDeploy) Run(ctx context.Context) {
|
|
go deploy.funcController.Run(ctx.Done())
|
|
}
|
|
|
|
func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controller) {
|
|
resyncPeriod := 30 * time.Second
|
|
listWatch := k8sCache.NewListWatchFromClient(deploy.crdClient, "functions", metav1.NamespaceDefault, fields.Everything())
|
|
store, controller := k8sCache.NewInformer(listWatch, &crd.Function{}, resyncPeriod, k8sCache.ResourceEventHandlerFuncs{
|
|
AddFunc: func(obj interface{}) {
|
|
fn := obj.(*crd.Function)
|
|
deploy.createFunction(fn)
|
|
},
|
|
DeleteFunc: func(obj interface{}) {
|
|
fn := obj.(*crd.Function)
|
|
deploy.deleteFunction(fn)
|
|
},
|
|
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
|
oldFn := oldObj.(*crd.Function)
|
|
newFn := newObj.(*crd.Function)
|
|
deploy.fnUpdate(oldFn, newFn)
|
|
},
|
|
})
|
|
return store, controller
|
|
}
|
|
|
|
func (deploy *NewDeploy) service() {
|
|
for {
|
|
req := <-deploy.requestChannel
|
|
switch req.reqType {
|
|
case FnCreate:
|
|
fsvc, err := deploy.fnCreate(req.fn)
|
|
req.responseChannel <- &fnResponse{
|
|
error: err,
|
|
fSvc: fsvc,
|
|
}
|
|
continue
|
|
case FnDelete:
|
|
_, err := deploy.fnDelete(req.fn)
|
|
req.responseChannel <- &fnResponse{
|
|
error: err,
|
|
fSvc: nil,
|
|
}
|
|
continue
|
|
// Update needs two inputs and will be called directly by controller
|
|
}
|
|
}
|
|
}
|
|
|
|
func (deploy *NewDeploy) GetFuncSvc(metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
|
|
c := make(chan *fnResponse)
|
|
fn, err := deploy.fissionClient.Functions(metadata.Namespace).Get(metadata.Name)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
fsvc, err := deploy.fsCache.GetByFunctionUID(metadata.UID)
|
|
|
|
// If the function service cache exists, means
|
|
// the kubeObjects of function are created before.
|
|
// In this case, return cached fsvc.
|
|
if err == nil {
|
|
return fsvc, nil
|
|
}
|
|
|
|
if !fscache.IsNotFoundError(err) {
|
|
log.Printf("error getting function service by uid: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
deploy.requestChannel <- &fnRequest{
|
|
fn: fn,
|
|
reqType: FnCreate,
|
|
responseChannel: c,
|
|
}
|
|
|
|
resp := <-c
|
|
if resp.error != nil {
|
|
return nil, resp.error
|
|
}
|
|
return resp.fSvc, nil
|
|
}
|
|
|
|
func (deploy *NewDeploy) createFunction(fn *crd.Function) {
|
|
if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy {
|
|
return
|
|
}
|
|
if fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale <= 0 {
|
|
return
|
|
}
|
|
// Eager creation of function if minScale is greater than 0
|
|
log.Printf("Eagerly creating newDeploy objects for function")
|
|
c := make(chan *fnResponse)
|
|
deploy.requestChannel <- &fnRequest{
|
|
fn: fn,
|
|
reqType: FnCreate,
|
|
responseChannel: c,
|
|
}
|
|
resp := <-c
|
|
if resp.error != nil {
|
|
log.Printf("Error eager creating function: %v", resp.error)
|
|
}
|
|
}
|
|
|
|
func (deploy *NewDeploy) updateFunction(fn *crd.Function) {
|
|
c := make(chan *fnResponse)
|
|
deploy.requestChannel <- &fnRequest{
|
|
fn: fn,
|
|
reqType: FnUpdate,
|
|
responseChannel: c,
|
|
}
|
|
resp := <-c
|
|
if resp.error != nil {
|
|
log.Printf("Error eager updating function: %v", resp.error)
|
|
}
|
|
}
|
|
|
|
func (deploy *NewDeploy) deleteFunction(fn *crd.Function) {
|
|
if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy {
|
|
c := make(chan *fnResponse)
|
|
deploy.requestChannel <- &fnRequest{
|
|
fn: fn,
|
|
reqType: FnDelete,
|
|
responseChannel: c,
|
|
}
|
|
resp := <-c
|
|
if resp.error != nil {
|
|
log.Printf("Error deleing the function: %v", resp.error)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (deploy *NewDeploy) fnCreate(fn *crd.Function) (*fscache.FuncSvc, error) {
|
|
fsvc, err := deploy.fsCache.GetByFunction(&fn.Metadata)
|
|
if err == nil {
|
|
return fsvc, err
|
|
}
|
|
|
|
env, err := deploy.fissionClient.
|
|
Environments(fn.Spec.Environment.Namespace).
|
|
Get(fn.Spec.Environment.Name)
|
|
if err != nil {
|
|
return fsvc, err
|
|
}
|
|
|
|
objName := deploy.getObjName(fn)
|
|
|
|
deployLabels := deploy.getDeployLabels(fn, env)
|
|
|
|
// Envoy(istio-proxy) returns 404 directly before istio pilot
|
|
// propagates latest Envoy-specific configuration.
|
|
// Since newdeploy waits for pods of deployment to be ready,
|
|
// change the order of kubeObject creation (create service first,
|
|
// then deployment) to take advantage of waiting time.
|
|
svc, err := deploy.createOrGetSvc(deployLabels, objName)
|
|
if err != nil {
|
|
log.Printf("Error creating the service %v: %v", objName, err)
|
|
return fsvc, err
|
|
}
|
|
svcAddress := fmt.Sprintf("%v.%v", svc.Name, svc.Namespace)
|
|
|
|
depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels)
|
|
if err != nil {
|
|
log.Printf("Error creating the deployment %v: %v", objName, err)
|
|
return fsvc, err
|
|
}
|
|
|
|
hpa, err := deploy.createOrGetHpa(objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl)
|
|
if err != nil {
|
|
return fsvc, errors.Wrap(err, fmt.Sprintf("error creating the HPA %v:", objName))
|
|
}
|
|
|
|
kubeObjRefs := []api.ObjectReference{
|
|
{
|
|
//obj.TypeMeta.Kind does not work hence this, needs investigationa and a fix
|
|
Kind: "deployment",
|
|
Name: depl.ObjectMeta.Name,
|
|
APIVersion: depl.TypeMeta.APIVersion,
|
|
Namespace: depl.ObjectMeta.Namespace,
|
|
ResourceVersion: depl.ObjectMeta.ResourceVersion,
|
|
UID: depl.ObjectMeta.UID,
|
|
},
|
|
{
|
|
Kind: "service",
|
|
Name: svc.ObjectMeta.Name,
|
|
APIVersion: svc.TypeMeta.APIVersion,
|
|
Namespace: svc.ObjectMeta.Namespace,
|
|
ResourceVersion: svc.ObjectMeta.ResourceVersion,
|
|
UID: svc.ObjectMeta.UID,
|
|
},
|
|
{
|
|
Kind: "horizontalpodautoscaler",
|
|
Name: hpa.ObjectMeta.Name,
|
|
APIVersion: hpa.TypeMeta.APIVersion,
|
|
Namespace: hpa.ObjectMeta.Namespace,
|
|
ResourceVersion: hpa.ObjectMeta.ResourceVersion,
|
|
UID: hpa.ObjectMeta.UID,
|
|
},
|
|
}
|
|
|
|
fsvc = &fscache.FuncSvc{
|
|
Name: objName,
|
|
Function: &fn.Metadata,
|
|
Environment: env,
|
|
Address: svcAddress,
|
|
KubernetesObjects: kubeObjRefs,
|
|
Executor: fscache.NEWDEPLOY,
|
|
}
|
|
|
|
_, err = deploy.fsCache.Add(*fsvc)
|
|
if err != nil {
|
|
log.Printf("Error adding the function to cache: %v", err)
|
|
return fsvc, err
|
|
}
|
|
return fsvc, nil
|
|
}
|
|
|
|
func (deploy *NewDeploy) fnUpdate(oldFn *crd.Function, newFn *crd.Function) {
|
|
|
|
if oldFn.Metadata.ResourceVersion == newFn.Metadata.ResourceVersion {
|
|
return
|
|
}
|
|
|
|
// Ignoring updates to functions which are not of NewDeployment type
|
|
if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy &&
|
|
oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy {
|
|
return
|
|
}
|
|
|
|
deployChanged := false
|
|
|
|
if oldFn.Spec.InvokeStrategy != newFn.Spec.InvokeStrategy {
|
|
|
|
// Executor type is no longer New Deployment
|
|
if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy &&
|
|
oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy {
|
|
log.Printf("function does not use new deployment executor anymore, deleting resources: %v", newFn)
|
|
// IMP - pass the oldFn, as the new/modified function is not in cache
|
|
deploy.fnDelete(oldFn)
|
|
return
|
|
}
|
|
|
|
// Executor type changed to New Deployment from something else
|
|
if oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fission.ExecutorTypeNewdeploy &&
|
|
newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy {
|
|
log.Printf("function type changed to new deployment, creating resources: %v", newFn)
|
|
_, err := deploy.fnCreate(newFn)
|
|
if err != nil {
|
|
updateStatus(oldFn, err, "error changing the function's type to newdeploy")
|
|
}
|
|
return
|
|
}
|
|
|
|
hpa, err := deploy.getHpa(newFn)
|
|
if err != nil {
|
|
updateStatus(oldFn, err, "error getting HPA while updating function")
|
|
return
|
|
}
|
|
|
|
hpaChanged := false
|
|
|
|
if newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale {
|
|
replicas := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
|
|
hpa.Spec.MinReplicas = &replicas
|
|
deployChanged = true
|
|
hpaChanged = true
|
|
}
|
|
|
|
if newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale {
|
|
hpa.Spec.MaxReplicas = int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale)
|
|
hpaChanged = true
|
|
}
|
|
|
|
if newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent != oldFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent {
|
|
targetCpupercent := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent)
|
|
hpa.Spec.TargetCPUUtilizationPercentage = &targetCpupercent
|
|
hpaChanged = true
|
|
}
|
|
|
|
if hpaChanged {
|
|
err := deploy.updateHpa(hpa)
|
|
if err != nil {
|
|
updateStatus(oldFn, err, "error updating HPA while updating function")
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
if oldFn.Spec.Environment != newFn.Spec.Environment ||
|
|
oldFn.Spec.Package.PackageRef != newFn.Spec.Package.PackageRef {
|
|
deployChanged = true
|
|
}
|
|
|
|
// If length of slice has changed then no need to check individual elements
|
|
if len(oldFn.Spec.Secrets) != len(newFn.Spec.Secrets) {
|
|
deployChanged = true
|
|
} else {
|
|
for i, newSecret := range newFn.Spec.Secrets {
|
|
if newSecret != oldFn.Spec.Secrets[i] {
|
|
deployChanged = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if len(oldFn.Spec.ConfigMaps) != len(newFn.Spec.ConfigMaps) {
|
|
deployChanged = true
|
|
} else {
|
|
for i, newConfig := range newFn.Spec.ConfigMaps {
|
|
if newConfig != oldFn.Spec.ConfigMaps[i] {
|
|
deployChanged = true
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if deployChanged == true {
|
|
env, err := deploy.fissionClient.Environments(newFn.Spec.Environment.Namespace).
|
|
Get(newFn.Spec.Environment.Name)
|
|
if err != nil {
|
|
updateStatus(oldFn, err, "failed to get environment while updating function")
|
|
return
|
|
}
|
|
deployName := deploy.getObjName(oldFn)
|
|
deployLabels := deploy.getDeployLabels(oldFn, env)
|
|
log.Printf("updating deployment due to function update")
|
|
newDeployment, err := deploy.getDeploymentSpec(newFn, env, deployName, deployLabels)
|
|
if err != nil {
|
|
updateStatus(oldFn, err, "failed to get new deployment spec while updating function")
|
|
return
|
|
}
|
|
err = deploy.updateDeployment(newDeployment)
|
|
if err != nil {
|
|
updateStatus(oldFn, err, "failed to update deployment while updating function")
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func (deploy *NewDeploy) fnDelete(fn *crd.Function) (*fscache.FuncSvc, error) {
|
|
|
|
var delError error
|
|
|
|
fsvc, err := deploy.fsCache.GetByFunction(&fn.Metadata)
|
|
if err != nil {
|
|
log.Printf("fsvc not fonud in cache: %v", fn.Metadata)
|
|
delError = err
|
|
return nil, err
|
|
}
|
|
|
|
_, err = deploy.fsCache.DeleteOld(fsvc, time.Second*0)
|
|
if err != nil {
|
|
log.Printf("Error deleting the function from cache: %v", fsvc)
|
|
delError = err
|
|
}
|
|
objName := fsvc.Name
|
|
|
|
err = deploy.deleteDeployment(deploy.namespace, objName)
|
|
if err != nil {
|
|
log.Printf("Error deleting the deployment: %v", objName)
|
|
delError = err
|
|
}
|
|
|
|
err = deploy.deleteSvc(deploy.namespace, objName)
|
|
if err != nil {
|
|
log.Printf("Error deleting the service: %v", objName)
|
|
delError = err
|
|
}
|
|
|
|
err = deploy.deleteHpa(deploy.namespace, objName)
|
|
if err != nil {
|
|
log.Printf("Error deleting the HPA: %v", objName)
|
|
delError = err
|
|
}
|
|
|
|
if delError != nil {
|
|
return nil, delError
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
func (deploy *NewDeploy) getObjName(fn *crd.Function) string {
|
|
return fmt.Sprintf("%v-%v",
|
|
fn.Metadata.Name,
|
|
deploy.instanceID)
|
|
}
|
|
|
|
func (deploy *NewDeploy) getDeployLabels(fn *crd.Function, env *crd.Environment) map[string]string {
|
|
return map[string]string{
|
|
"environmentName": env.Metadata.Name,
|
|
"environmentUid": string(env.Metadata.UID),
|
|
"functionName": fn.Metadata.Name,
|
|
"functionUid": string(fn.Metadata.UID),
|
|
fission.EXECUTOR_INSTANCEID_LABEL: deploy.instanceID,
|
|
"executorType": fission.ExecutorTypeNewdeploy,
|
|
}
|
|
}
|
|
|
|
// updateKubeObjRefRV update the resource version of kubeObjectRef with
|
|
// given kind and return error if failed to find the reference.
|
|
func (deploy *NewDeploy) updateKubeObjRefRV(fsvc *fscache.FuncSvc, objKind string, rv string) error {
|
|
kubeObjs := fsvc.KubernetesObjects
|
|
for i, obj := range kubeObjs {
|
|
if obj.Kind == objKind {
|
|
kubeObjs[i].ResourceVersion = rv
|
|
return nil
|
|
}
|
|
}
|
|
fsvc.KubernetesObjects = kubeObjs
|
|
return errors.New(fmt.Sprintf("error finding kubernetes object reference with kind: %v", objKind))
|
|
}
|
|
|
|
// updateStatus is a function which updates status of update.
|
|
// Current implementation only logs messages, in future it will update function status
|
|
func updateStatus(fn *crd.Function, err error, message string) {
|
|
log.Printf(message, err)
|
|
}
|