Fix executor tries to create a new deployment when a function is updated (#524)

Newdeploy manager now checks the existence of function service cache by function UID before trying to create a new deployment. And return the cached fsvc directly if the cache exists.
2018-03-08 02:52:36 +08:00
parent a3826046a5
commit b4300feabc
7 changed files with 169 additions and 45 deletions
@@ -58,7 +58,10 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro
 	}

 	existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deploy.namespace).Get(deployName, metav1.GetOptions{})
-	if err == nil && existingDepl.Status.ReadyReplicas >= replicas {
+	if err == nil {
+		if existingDepl.Status.ReadyReplicas < replicas {
+			existingDepl, err = deploy.waitForDeploy(existingDepl, replicas)
+		}
 		return existingDepl, err
 	}

@@ -75,18 +78,7 @@ func (deploy *NewDeploy) createOrGetDeployment(fn *crd.Function, env *crd.Enviro
 			return nil, err
 		}

-		for i := 0; i < 120; i++ {
-			latestDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deploy.namespace).Get(depl.Name, metav1.GetOptions{})
-			if err != nil {
-				return nil, err
-			}
-			//TODO check for imagePullerror
-			if latestDepl.Status.ReadyReplicas == replicas {
-				return latestDepl, err
-			}
-			time.Sleep(time.Second)
-		}
-		return nil, errors.New("failed to create deployment within timeout window")
+		return deploy.waitForDeploy(depl, replicas)
 	}

 	return nil, err
@@ -459,3 +451,18 @@ func (deploy *NewDeploy) deleteSvc(ns string, name string) error {
 	}
 	return nil
 }
+
+func (deploy *NewDeploy) waitForDeploy(depl *v1beta1.Deployment, replicas int32) (*v1beta1.Deployment, error) {
+	for i := 0; i < 120; i++ {
+		latestDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deploy.namespace).Get(depl.Name, metav1.GetOptions{})
+		if err != nil {
+			return nil, err
+		}
+		//TODO check for imagePullerror
+		if latestDepl.Status.ReadyReplicas >= replicas {
+			return latestDepl, err
+		}
+		time.Sleep(time.Second)
+	}
+	return nil, errors.New("failed to create deployment within timeout window")
+}
@@ -77,6 +77,7 @@ type (

 const (
 	FnCreate requestType = iota
+	FnUpdate
 	FnDelete
 )

@@ -191,11 +192,27 @@ func (deploy *NewDeploy) GetFuncSvc(metadata *metav1.ObjectMeta) (*fscache.FuncS
 	if err != nil {
 		return nil, err
 	}
+
+	fsvc, err := deploy.fsCache.GetByFunctionUID(metadata.UID)
+
+	// If the function service cache exists, means
+	// the kubeObjects of function are created before.
+	// In this case, return cached fsvc.
+	if err == nil {
+		return fsvc, nil
+	}
+
+	if !fscache.IsNotFoundError(err) {
+		log.Printf("error getting function service by uid: %v", err)
+		return nil, err
+	}
+
 	deploy.requestChannel <- &fnRequest{
 		fn:              fn,
 		reqType:         FnCreate,
 		responseChannel: c,
 	}
+
 	resp := <-c
 	if resp.error != nil {
 		return nil, resp.error
@@ -224,6 +241,19 @@ func (deploy *NewDeploy) createFunction(fn *crd.Function) {
 	}
 }

+func (deploy *NewDeploy) updateFunction(fn *crd.Function) {
+	c := make(chan *fnResponse)
+	deploy.requestChannel <- &fnRequest{
+		fn:              fn,
+		reqType:         FnUpdate,
+		responseChannel: c,
+	}
+	resp := <-c
+	if resp.error != nil {
+		log.Printf("Error eager updating function: %v", resp.error)
+	}
+}
+
 func (deploy *NewDeploy) deleteFunction(fn *crd.Function) {
 	if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fission.ExecutorTypeNewdeploy {
 		c := make(chan *fnResponse)
@@ -336,7 +366,7 @@ func (deploy *NewDeploy) fnUpdate(oldFn *crd.Function, newFn *crd.Function) {
 		return
 	}

-	changed := false
+	deployChanged := false

 	if oldFn.Spec.InvokeStrategy != newFn.Spec.InvokeStrategy {

@@ -366,59 +396,63 @@ func (deploy *NewDeploy) fnUpdate(oldFn *crd.Function, newFn *crd.Function) {
 			return
 		}

+		hpaChanged := false
+
 		if newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale {
 			replicas := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
 			hpa.Spec.MinReplicas = &replicas
-			changed = true // Will start deployment update
+			deployChanged = true
+			hpaChanged = true
 		}

 		if newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale {
 			hpa.Spec.MaxReplicas = int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale)
+			hpaChanged = true
 		}

 		if newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent != oldFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent {
 			targetCpupercent := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent)
 			hpa.Spec.TargetCPUUtilizationPercentage = &targetCpupercent
+			hpaChanged = true
 		}

-		err = deploy.updateHpa(hpa)
-		if err != nil {
-			updateStatus(oldFn, err, "error updating HPA while updating function")
-			return
+		if hpaChanged {
+			err := deploy.updateHpa(hpa)
+			if err != nil {
+				updateStatus(oldFn, err, "error updating HPA while updating function")
+				return
+			}
 		}
 	}

-	if oldFn.Spec.Environment != newFn.Spec.Environment {
-		changed = true
-	}
-
-	if oldFn.Spec.Package.PackageRef != newFn.Spec.Package.PackageRef {
-		changed = true
+	if oldFn.Spec.Environment != newFn.Spec.Environment ||
+		oldFn.Spec.Package.PackageRef != newFn.Spec.Package.PackageRef {
+		deployChanged = true
 	}

 	// If length of slice has changed then no need to check individual elements
 	if len(oldFn.Spec.Secrets) != len(newFn.Spec.Secrets) {
-		changed = true
+		deployChanged = true
 	} else {
 		for i, newSecret := range newFn.Spec.Secrets {
 			if newSecret != oldFn.Spec.Secrets[i] {
-				changed = true
+				deployChanged = true
 				break
 			}
 		}
 	}
 	if len(oldFn.Spec.ConfigMaps) != len(newFn.Spec.ConfigMaps) {
-		changed = true
+		deployChanged = true
 	} else {
 		for i, newConfig := range newFn.Spec.ConfigMaps {
 			if newConfig != oldFn.Spec.ConfigMaps[i] {
-				changed = true
+				deployChanged = true
 				break
 			}
 		}
 	}

-	if changed == true {
+	if deployChanged == true {
 		env, err := deploy.fissionClient.Environments(newFn.Spec.Environment.Namespace).
 			Get(newFn.Spec.Environment.Name)
 		if err != nil {
@@ -438,7 +472,6 @@ func (deploy *NewDeploy) fnUpdate(oldFn *crd.Function, newFn *crd.Function) {
 			updateStatus(oldFn, err, "failed to update deployment while updating function")
 			return
 		}
-		return
 	}
 }

@@ -501,6 +534,20 @@ func (deploy *NewDeploy) getDeployLabels(fn *crd.Function, env *crd.Environment)
 	}
 }

+// updateKubeObjRefRV update the resource version of kubeObjectRef with
+// given kind and return error if failed to find the reference.
+func (deploy *NewDeploy) updateKubeObjRefRV(fsvc *fscache.FuncSvc, objKind string, rv string) error {
+	kubeObjs := fsvc.KubernetesObjects
+	for i, obj := range kubeObjs {
+		if obj.Kind == objKind {
+			kubeObjs[i].ResourceVersion = rv
+			return nil
+		}
+	}
+	fsvc.KubernetesObjects = kubeObjs
+	return errors.New(fmt.Sprintf("error finding kubernetes object reference with kind: %v", objKind))
+}
+
 // updateStatus is a function which updates status of update.
 // Current implementation only logs messages, in future it will update function status
 func updateStatus(fn *crd.Function, err error, message string) {