Move packages to proejct/pkg to follow go project folder structure convention (#1190)
This commit is contained in:
Ta-Ching Chen
@@ -0,0 +1,447 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package newdeploy
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/fission/fission/pkg/types"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"go.uber.org/zap"
|
||||
asv1 "k8s.io/api/autoscaling/v1"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
"k8s.io/api/extensions/v1beta1"
|
||||
k8s_err "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/executor/util"
|
||||
)
|
||||
|
||||
const (
|
||||
DeploymentKind = "Deployment"
|
||||
DeploymentVersion = "extensions/v1beta1"
|
||||
)
|
||||
|
||||
func (deploy *NewDeploy) createOrGetDeployment(fn *fv1.Function, env *fv1.Environment,
|
||||
deployName string, deployLabels map[string]string, deployNamespace string, firstcreate bool) (*v1beta1.Deployment, error) {
|
||||
|
||||
minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
|
||||
|
||||
// If it's not the first time creation and minscale is 0 means that all pods for function were recycled,
|
||||
// in such cases we need set minscale to 1 for router to serve requests.
|
||||
if !firstcreate && minScale <= 0 {
|
||||
minScale = 1
|
||||
}
|
||||
|
||||
waitForDeploy := minScale > 0
|
||||
|
||||
existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Get(deployName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
if waitForDeploy {
|
||||
err = deploy.scaleDeployment(existingDepl.Namespace, existingDepl.Name, minScale)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error scaling up function deployment", zap.Error(err), zap.String("function", fn.Metadata.Name))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if existingDepl.Status.AvailableReplicas < minScale {
|
||||
existingDepl, err = deploy.waitForDeploy(existingDepl, minScale)
|
||||
}
|
||||
}
|
||||
return existingDepl, err
|
||||
}
|
||||
|
||||
if err != nil && k8s_err.IsNotFound(err) {
|
||||
err := deploy.setupRBACObjs(deployNamespace, fn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
deployment, err := deploy.getDeploymentSpec(fn, env, deployName, deployLabels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
depl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Create(deployment)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error while creating function deployment",
|
||||
zap.Error(err),
|
||||
zap.String("function", fn.Metadata.Name),
|
||||
zap.String("deployment_name", deployName),
|
||||
zap.String("deployment_namespace", deployNamespace))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if waitForDeploy {
|
||||
depl, err = deploy.waitForDeploy(depl, minScale)
|
||||
}
|
||||
|
||||
return depl, err
|
||||
}
|
||||
|
||||
return nil, err
|
||||
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) setupRBACObjs(deployNamespace string, fn *fv1.Function) error {
|
||||
// create fetcher SA in this ns, if not already created
|
||||
err := deploy.fetcherConfig.SetupServiceAccount(deploy.kubernetesClient, deployNamespace, fn.Metadata)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating fission fetcher service account for function",
|
||||
zap.Error(err),
|
||||
zap.String("service_account_name", types.FissionFetcherSA),
|
||||
zap.String("service_account_namespace", deployNamespace),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
return err
|
||||
}
|
||||
|
||||
// create a cluster role binding for the fetcher SA, if not already created, granting access to do a get on packages in any ns
|
||||
err = utils.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, types.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, types.PackageGetterCR, types.ClusterRole, types.FissionFetcherSA, deployNamespace)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating role binding for function",
|
||||
zap.Error(err),
|
||||
zap.String("role_binding", types.PackageGetterRB),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
return err
|
||||
}
|
||||
|
||||
// create rolebinding in function namespace for fetcherSA.envNamespace to be able to get secrets and configmaps
|
||||
err = utils.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, types.SecretConfigMapGetterRB, fn.Metadata.Namespace, types.SecretConfigMapGetterCR, types.ClusterRole, types.FissionFetcherSA, deployNamespace)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating role binding for function",
|
||||
zap.Error(err),
|
||||
zap.String("role_binding", types.SecretConfigMapGetterRB),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
return err
|
||||
}
|
||||
|
||||
deploy.logger.Info("set up all RBAC objects for function",
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getDeployment(ns, name string) (*v1beta1.Deployment, error) {
|
||||
return deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Get(name, metav1.GetOptions{})
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) updateDeployment(deployment *v1beta1.Deployment, ns string) error {
|
||||
_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Update(deployment)
|
||||
return err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) deleteDeployment(ns string, name string) error {
|
||||
// DeletePropagationBackground deletes the object immediately and dependent are deleted later
|
||||
// DeletePropagationForeground not advisable; it markes for deleteion and API can still serve those objects
|
||||
deletePropagation := metav1.DeletePropagationBackground
|
||||
err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Delete(name, &metav1.DeleteOptions{
|
||||
PropagationPolicy: &deletePropagation,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getDeploymentSpec(fn *fv1.Function, env *fv1.Environment,
|
||||
deployName string, deployLabels map[string]string) (*v1beta1.Deployment, error) {
|
||||
|
||||
replicas := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
|
||||
|
||||
gracePeriodSeconds := int64(6 * 60)
|
||||
if env.Spec.TerminationGracePeriod > 0 {
|
||||
gracePeriodSeconds = env.Spec.TerminationGracePeriod
|
||||
}
|
||||
|
||||
podAnnotations := env.Metadata.Annotations
|
||||
if podAnnotations == nil {
|
||||
podAnnotations = make(map[string]string)
|
||||
}
|
||||
if deploy.useIstio && env.Spec.AllowAccessToExternalNetwork {
|
||||
podAnnotations["sidecar.istio.io/inject"] = "false"
|
||||
}
|
||||
resources := deploy.getResources(env, fn)
|
||||
|
||||
deployment := &v1beta1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: deployName,
|
||||
Labels: deployLabels,
|
||||
},
|
||||
Spec: v1beta1.DeploymentSpec{
|
||||
Replicas: &replicas,
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: deployLabels,
|
||||
},
|
||||
Template: apiv1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: deployLabels,
|
||||
Annotations: podAnnotations,
|
||||
},
|
||||
Spec: apiv1.PodSpec{
|
||||
Containers: []apiv1.Container{
|
||||
util.MergeContainerSpecs(&apiv1.Container{
|
||||
Name: fn.Metadata.Name,
|
||||
Image: env.Spec.Runtime.Image,
|
||||
ImagePullPolicy: deploy.runtimeImagePullPolicy,
|
||||
TerminationMessagePath: "/dev/termination-log",
|
||||
Lifecycle: &apiv1.Lifecycle{
|
||||
PreStop: &apiv1.Handler{
|
||||
Exec: &apiv1.ExecAction{
|
||||
Command: []string{
|
||||
"/bin/sleep",
|
||||
fmt.Sprintf("%v", gracePeriodSeconds),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Resources: resources,
|
||||
}, env.Spec.Runtime.Container),
|
||||
},
|
||||
ServiceAccountName: "fission-fetcher",
|
||||
TerminationGracePeriodSeconds: &gracePeriodSeconds,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Order of merging is important here - first fetcher, then containers and lastly pod spec
|
||||
err := deploy.fetcherConfig.AddSpecializingFetcherToPodSpec(
|
||||
&deployment.Spec.Template.Spec,
|
||||
fn.Metadata.Name,
|
||||
fn,
|
||||
env,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if env.Spec.Runtime.PodSpec != nil {
|
||||
err := util.MergePodSpec(&deployment.Spec.Template.Spec, env.Spec.Runtime.PodSpec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return deployment, nil
|
||||
}
|
||||
|
||||
// getResources overrides only the resources which are overridden at function level otherwise
|
||||
// default to resources specified at environment level
|
||||
func (deploy *NewDeploy) getResources(env *fv1.Environment, fn *fv1.Function) apiv1.ResourceRequirements {
|
||||
resources := env.Spec.Resources
|
||||
if resources.Requests == nil {
|
||||
resources.Requests = make(map[apiv1.ResourceName]resource.Quantity)
|
||||
}
|
||||
if resources.Limits == nil {
|
||||
resources.Limits = make(map[apiv1.ResourceName]resource.Quantity)
|
||||
}
|
||||
// Only override the once specified at function, rest default to values from env.
|
||||
val, ok := fn.Spec.Resources.Requests[apiv1.ResourceCPU]
|
||||
if ok && !val.IsZero() {
|
||||
resources.Requests[apiv1.ResourceCPU] = fn.Spec.Resources.Requests[apiv1.ResourceCPU]
|
||||
}
|
||||
|
||||
val, ok = fn.Spec.Resources.Requests[apiv1.ResourceMemory]
|
||||
if ok && !val.IsZero() {
|
||||
resources.Requests[apiv1.ResourceMemory] = fn.Spec.Resources.Requests[apiv1.ResourceMemory]
|
||||
}
|
||||
|
||||
val, ok = fn.Spec.Resources.Limits[apiv1.ResourceCPU]
|
||||
if ok && !val.IsZero() {
|
||||
resources.Limits[apiv1.ResourceCPU] = fn.Spec.Resources.Limits[apiv1.ResourceCPU]
|
||||
}
|
||||
|
||||
val, ok = fn.Spec.Resources.Limits[apiv1.ResourceMemory]
|
||||
if ok && !val.IsZero() {
|
||||
resources.Limits[apiv1.ResourceMemory] = fn.Spec.Resources.Limits[apiv1.ResourceMemory]
|
||||
}
|
||||
|
||||
return resources
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) createOrGetHpa(hpaName string, execStrategy *fv1.ExecutionStrategy, depl *v1beta1.Deployment) (*asv1.HorizontalPodAutoscaler, error) {
|
||||
|
||||
minRepl := int32(execStrategy.MinScale)
|
||||
if minRepl == 0 {
|
||||
minRepl = 1
|
||||
}
|
||||
maxRepl := int32(execStrategy.MaxScale)
|
||||
if maxRepl == 0 {
|
||||
maxRepl = minRepl
|
||||
}
|
||||
targetCPU := int32(execStrategy.TargetCPUPercent)
|
||||
|
||||
existingHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Get(hpaName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
return existingHpa, err
|
||||
}
|
||||
|
||||
if depl == nil {
|
||||
return nil, errors.New("failed to create HPA, found empty deployment")
|
||||
}
|
||||
|
||||
if err != nil && k8s_err.IsNotFound(err) {
|
||||
hpa := asv1.HorizontalPodAutoscaler{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: hpaName,
|
||||
Labels: depl.Labels,
|
||||
},
|
||||
Spec: asv1.HorizontalPodAutoscalerSpec{
|
||||
ScaleTargetRef: asv1.CrossVersionObjectReference{
|
||||
Kind: DeploymentKind,
|
||||
Name: depl.ObjectMeta.Name,
|
||||
APIVersion: DeploymentVersion,
|
||||
},
|
||||
MinReplicas: &minRepl,
|
||||
MaxReplicas: maxRepl,
|
||||
TargetCPUUtilizationPercentage: &targetCPU,
|
||||
},
|
||||
}
|
||||
|
||||
cHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Create(&hpa)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cHpa, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getHpa(ns, name string) (*asv1.HorizontalPodAutoscaler, error) {
|
||||
return deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(ns).Get(name, metav1.GetOptions{})
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) updateHpa(hpa *asv1.HorizontalPodAutoscaler) error {
|
||||
_, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Update(hpa)
|
||||
return err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) deleteHpa(ns string, name string) error {
|
||||
err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(ns).Delete(name, &metav1.DeleteOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) createOrGetSvc(deployLabels map[string]string, svcName string, svcNamespace string) (*apiv1.Service, error) {
|
||||
|
||||
existingSvc, err := deploy.kubernetesClient.CoreV1().Services(svcNamespace).Get(svcName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
return existingSvc, err
|
||||
}
|
||||
|
||||
if err != nil && k8s_err.IsNotFound(err) {
|
||||
service := &apiv1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: svcName,
|
||||
Labels: deployLabels,
|
||||
},
|
||||
Spec: apiv1.ServiceSpec{
|
||||
Ports: []apiv1.ServicePort{
|
||||
{
|
||||
Name: "runtime-env-port",
|
||||
Port: int32(80),
|
||||
TargetPort: intstr.FromInt(8888),
|
||||
},
|
||||
{
|
||||
Name: "fetcher-port",
|
||||
Port: int32(8000),
|
||||
TargetPort: intstr.FromInt(8000),
|
||||
},
|
||||
},
|
||||
Selector: deployLabels,
|
||||
Type: apiv1.ServiceTypeClusterIP,
|
||||
},
|
||||
}
|
||||
|
||||
svc, err := deploy.kubernetesClient.CoreV1().Services(svcNamespace).Create(service)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) deleteSvc(ns string, name string) error {
|
||||
err := deploy.kubernetesClient.CoreV1().Services(ns).Delete(name, &metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) waitForDeploy(depl *v1beta1.Deployment, replicas int32) (*v1beta1.Deployment, error) {
|
||||
for i := 0; i < 120; i++ {
|
||||
latestDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(depl.ObjectMeta.Namespace).Get(depl.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
//TODO check for imagePullerror
|
||||
// use AvailableReplicas here is better than ReadyReplicas
|
||||
// since the pods may not be able to serve network traffic yet.
|
||||
if latestDepl.Status.AvailableReplicas >= replicas {
|
||||
return latestDepl, err
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
return nil, errors.New("failed to create deployment within timeout window")
|
||||
}
|
||||
|
||||
// cleanupNewdeploy cleans all kubernetes objects related to function
|
||||
func (deploy *NewDeploy) cleanupNewdeploy(ns string, name string) error {
|
||||
var multierr *multierror.Error
|
||||
|
||||
err := deploy.deleteSvc(ns, name)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error deleting service for newdeploy function",
|
||||
zap.Error(err),
|
||||
zap.String("function_name", name),
|
||||
zap.String("function_namespace", ns))
|
||||
multierror.Append(multierr, err)
|
||||
}
|
||||
|
||||
err = deploy.deleteHpa(ns, name)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error deleting service for newdeploy function",
|
||||
zap.Error(err),
|
||||
zap.String("function_name", name),
|
||||
zap.String("function_namespace", ns))
|
||||
multierror.Append(multierr, err)
|
||||
}
|
||||
|
||||
err = deploy.deleteDeployment(ns, name)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error deleting deployment for newdeploy function",
|
||||
zap.Error(err),
|
||||
zap.String("function_name", name),
|
||||
zap.String("function_namespace", ns))
|
||||
multierror.Append(multierr, err)
|
||||
}
|
||||
return multierr.ErrorOrNil()
|
||||
}
|
||||
@@ -0,0 +1,722 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package newdeploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dchest/uniuri"
|
||||
"github.com/fission/fission/pkg/throttler"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/pkg/errors"
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
"k8s.io/api/extensions/v1beta1"
|
||||
k8sErrs "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
k8sTypes "k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
"github.com/fission/fission/pkg/executor/fscache"
|
||||
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
|
||||
"github.com/fission/fission/pkg/types"
|
||||
)
|
||||
|
||||
type (
|
||||
NewDeploy struct {
|
||||
logger *zap.Logger
|
||||
|
||||
kubernetesClient *kubernetes.Clientset
|
||||
fissionClient *crd.FissionClient
|
||||
crdClient *rest.RESTClient
|
||||
instanceID string
|
||||
fetcherConfig *fetcherConfig.Config
|
||||
|
||||
runtimeImagePullPolicy apiv1.PullPolicy
|
||||
namespace string
|
||||
useIstio bool
|
||||
collectorEndpoint string
|
||||
|
||||
fsCache *fscache.FunctionServiceCache // cache funcSvc's by function, address and pod name
|
||||
|
||||
throttler *throttler.Throttler
|
||||
funcStore k8sCache.Store
|
||||
funcController k8sCache.Controller
|
||||
|
||||
envStore k8sCache.Store
|
||||
envController k8sCache.Controller
|
||||
|
||||
idlePodReapTime time.Duration
|
||||
}
|
||||
)
|
||||
|
||||
func MakeNewDeploy(
|
||||
logger *zap.Logger,
|
||||
fissionClient *crd.FissionClient,
|
||||
kubernetesClient *kubernetes.Clientset,
|
||||
crdClient *rest.RESTClient,
|
||||
namespace string,
|
||||
fetcherConfig *fetcherConfig.Config,
|
||||
instanceID string,
|
||||
) *NewDeploy {
|
||||
|
||||
logger.Info("creating NewDeploy ExecutorType")
|
||||
|
||||
enableIstio := false
|
||||
if len(os.Getenv("ENABLE_ISTIO")) > 0 {
|
||||
istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
|
||||
if err != nil {
|
||||
logger.Error("failed to parse 'ENABLE_ISTIO', set to false", zap.Error(err))
|
||||
}
|
||||
enableIstio = istio
|
||||
}
|
||||
|
||||
nd := &NewDeploy{
|
||||
logger: logger.Named("new_deploy"),
|
||||
|
||||
fissionClient: fissionClient,
|
||||
kubernetesClient: kubernetesClient,
|
||||
crdClient: crdClient,
|
||||
instanceID: instanceID,
|
||||
|
||||
namespace: namespace,
|
||||
fsCache: fscache.MakeFunctionServiceCache(logger),
|
||||
throttler: throttler.MakeThrottler(1 * time.Minute),
|
||||
|
||||
fetcherConfig: fetcherConfig,
|
||||
runtimeImagePullPolicy: utils.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY")),
|
||||
useIstio: enableIstio,
|
||||
|
||||
idlePodReapTime: 2 * time.Minute,
|
||||
}
|
||||
|
||||
if nd.crdClient != nil {
|
||||
fnStore, fnController := nd.initFuncController()
|
||||
nd.funcStore = fnStore
|
||||
nd.funcController = fnController
|
||||
|
||||
envStore, envController := nd.initEnvController()
|
||||
nd.envStore = envStore
|
||||
nd.envController = envController
|
||||
}
|
||||
|
||||
return nd
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) Run(ctx context.Context) {
|
||||
//go deploy.service()
|
||||
go deploy.funcController.Run(ctx.Done())
|
||||
go deploy.envController.Run(ctx.Done())
|
||||
go deploy.idleObjectReaper()
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controller) {
|
||||
resyncPeriod := 30 * time.Second
|
||||
listWatch := k8sCache.NewListWatchFromClient(deploy.crdClient, "functions", metav1.NamespaceAll, fields.Everything())
|
||||
store, controller := k8sCache.NewInformer(listWatch, &fv1.Function{}, resyncPeriod, k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
fn := obj.(*fv1.Function)
|
||||
_, err := deploy.createFunction(fn, true)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error eager creating function",
|
||||
zap.Error(err),
|
||||
zap.Any("function", fn))
|
||||
}
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
fn := obj.(*fv1.Function)
|
||||
err := deploy.deleteFunction(fn)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error deleting function",
|
||||
zap.Error(err),
|
||||
zap.Any("function", fn))
|
||||
}
|
||||
},
|
||||
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
||||
oldFn := oldObj.(*fv1.Function)
|
||||
newFn := newObj.(*fv1.Function)
|
||||
err := deploy.updateFunction(oldFn, newFn)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error updating function",
|
||||
zap.Error(err),
|
||||
zap.Any("old_function", oldFn),
|
||||
zap.Any("new_function", newFn))
|
||||
}
|
||||
},
|
||||
})
|
||||
return store, controller
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) initEnvController() (k8sCache.Store, k8sCache.Controller) {
|
||||
resyncPeriod := 30 * time.Second
|
||||
listWatch := k8sCache.NewListWatchFromClient(deploy.crdClient, "environments", metav1.NamespaceAll, fields.Everything())
|
||||
store, controller := k8sCache.NewInformer(listWatch, &fv1.Environment{}, resyncPeriod, k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {},
|
||||
DeleteFunc: func(obj interface{}) {},
|
||||
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
||||
newEnv := newObj.(*fv1.Environment)
|
||||
oldEnv := oldObj.(*fv1.Environment)
|
||||
// Currently only an image update in environment calls for function's deployment recreation. In future there might be more attributes which would want to do it
|
||||
if oldEnv.Spec.Runtime.Image != newEnv.Spec.Runtime.Image {
|
||||
deploy.logger.Info("Updating all function of the environment that changed, old env:", zap.Any("environment", oldEnv))
|
||||
funcs := deploy.getEnvFunctions(&newEnv.Metadata)
|
||||
for _, f := range funcs {
|
||||
function, err := deploy.fissionClient.Functions(f.Metadata.Namespace).Get(f.Metadata.Name)
|
||||
if err != nil {
|
||||
deploy.logger.Error("Error getting function", zap.Error(err), zap.Any("function", function))
|
||||
}
|
||||
err = deploy.updateFuncDeployment(function, newEnv)
|
||||
if err != nil {
|
||||
deploy.logger.Error("Error updating function", zap.Error(err), zap.Any("function", function))
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
return store, controller
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getEnvFunctions(m *metav1.ObjectMeta) []fv1.Function {
|
||||
funcList, err := deploy.fissionClient.Functions(m.Namespace).List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Error("Error getting functions for env", zap.Error(err), zap.Any("environment", m))
|
||||
}
|
||||
relatedFunctions := make([]fv1.Function, 0)
|
||||
for _, f := range funcList.Items {
|
||||
if (f.Spec.Environment.Name == m.Name) && (f.Spec.Environment.Namespace == m.Namespace) {
|
||||
relatedFunctions = append(relatedFunctions, f)
|
||||
}
|
||||
}
|
||||
return relatedFunctions
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
|
||||
fn, err := deploy.fissionClient.Functions(metadata.Namespace).Get(metadata.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return deploy.createFunction(fn, false)
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) createFunction(fn *fv1.Function, firstcreate bool) (*fscache.FuncSvc, error) {
|
||||
if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
fsvcObj, err := deploy.throttler.RunOnce(string(fn.Metadata.UID), func(ableToCreate bool) (interface{}, error) {
|
||||
if ableToCreate {
|
||||
return deploy.fnCreate(fn, firstcreate)
|
||||
}
|
||||
return deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
|
||||
})
|
||||
|
||||
fsvc, ok := fsvcObj.(*fscache.FuncSvc)
|
||||
if !ok {
|
||||
deploy.logger.Panic("receive unknown object while creating function - expected pointer of function service object")
|
||||
}
|
||||
|
||||
return fsvc, err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) deleteFunction(fn *fv1.Function) error {
|
||||
if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
|
||||
return nil
|
||||
}
|
||||
err := deploy.fnDelete(fn)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "error deleting kubernetes objects of function %v", fn.Metadata)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) fnCreate(fn *fv1.Function, firstcreate bool) (*fscache.FuncSvc, error) {
|
||||
env, err := deploy.fissionClient.
|
||||
Environments(fn.Spec.Environment.Namespace).
|
||||
Get(fn.Spec.Environment.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objName := deploy.getObjName(fn)
|
||||
if !firstcreate {
|
||||
// retrieve back the previous obj name for later use.
|
||||
fsvc, err := deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
|
||||
if err == nil {
|
||||
objName = fsvc.Name
|
||||
}
|
||||
}
|
||||
deployLabels := deploy.getDeployLabels(fn, env)
|
||||
|
||||
// to support backward compatibility, if the function was created in default ns, we fall back to creating the
|
||||
// deployment of the function in fission-function ns
|
||||
ns := deploy.namespace
|
||||
if fn.Metadata.Namespace != metav1.NamespaceDefault {
|
||||
ns = fn.Metadata.Namespace
|
||||
}
|
||||
|
||||
// Envoy(istio-proxy) returns 404 directly before istio pilot
|
||||
// propagates latest Envoy-specific configuration.
|
||||
// Since newdeploy waits for pods of deployment to be ready,
|
||||
// change the order of kubeObject creation (create service first,
|
||||
// then deployment) to take advantage of waiting time.
|
||||
svc, err := deploy.createOrGetSvc(deployLabels, objName, ns)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating service", zap.Error(err), zap.String("service", objName))
|
||||
go deploy.cleanupNewdeploy(ns, objName)
|
||||
return nil, errors.Wrapf(err, "error creating service %v", objName)
|
||||
}
|
||||
svcAddress := fmt.Sprintf("%v.%v", svc.Name, svc.Namespace)
|
||||
depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels, ns, firstcreate)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating deployment", zap.Error(err), zap.String("deployment", objName))
|
||||
go deploy.cleanupNewdeploy(ns, objName)
|
||||
return nil, errors.Wrapf(err, "error creating deployment %v", objName)
|
||||
}
|
||||
|
||||
hpa, err := deploy.createOrGetHpa(objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName))
|
||||
go deploy.cleanupNewdeploy(ns, objName)
|
||||
return nil, errors.Wrapf(err, "error creating the HPA %v", objName)
|
||||
}
|
||||
|
||||
kubeObjRefs := []apiv1.ObjectReference{
|
||||
{
|
||||
//obj.TypeMeta.Kind does not work hence this, needs investigation and a fix
|
||||
Kind: "deployment",
|
||||
Name: depl.ObjectMeta.Name,
|
||||
APIVersion: depl.TypeMeta.APIVersion,
|
||||
Namespace: depl.ObjectMeta.Namespace,
|
||||
ResourceVersion: depl.ObjectMeta.ResourceVersion,
|
||||
UID: depl.ObjectMeta.UID,
|
||||
},
|
||||
{
|
||||
Kind: "service",
|
||||
Name: svc.ObjectMeta.Name,
|
||||
APIVersion: svc.TypeMeta.APIVersion,
|
||||
Namespace: svc.ObjectMeta.Namespace,
|
||||
ResourceVersion: svc.ObjectMeta.ResourceVersion,
|
||||
UID: svc.ObjectMeta.UID,
|
||||
},
|
||||
{
|
||||
Kind: "horizontalpodautoscaler",
|
||||
Name: hpa.ObjectMeta.Name,
|
||||
APIVersion: hpa.TypeMeta.APIVersion,
|
||||
Namespace: hpa.ObjectMeta.Namespace,
|
||||
ResourceVersion: hpa.ObjectMeta.ResourceVersion,
|
||||
UID: hpa.ObjectMeta.UID,
|
||||
},
|
||||
}
|
||||
|
||||
fsvc := &fscache.FuncSvc{
|
||||
Name: objName,
|
||||
Function: &fn.Metadata,
|
||||
Environment: env,
|
||||
Address: svcAddress,
|
||||
KubernetesObjects: kubeObjRefs,
|
||||
Executor: fscache.NEWDEPLOY,
|
||||
}
|
||||
|
||||
_, err = deploy.fsCache.Add(*fsvc)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error adding function to cache", zap.Error(err), zap.Any("function", fsvc.Function))
|
||||
return fsvc, err
|
||||
}
|
||||
return fsvc, nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) updateFunction(oldFn *fv1.Function, newFn *fv1.Function) error {
|
||||
|
||||
if oldFn.Metadata.ResourceVersion == newFn.Metadata.ResourceVersion {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ignoring updates to functions which are not of NewDeployment type
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy &&
|
||||
oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Executor type is no longer New Deployment
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy &&
|
||||
oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypeNewdeploy {
|
||||
deploy.logger.Info("function does not use new deployment executor anymore, deleting resources",
|
||||
zap.Any("function", newFn))
|
||||
// IMP - pass the oldFn, as the new/modified function is not in cache
|
||||
return deploy.deleteFunction(oldFn)
|
||||
}
|
||||
|
||||
// Executor type changed to New Deployment from something else
|
||||
if oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy &&
|
||||
newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypeNewdeploy {
|
||||
deploy.logger.Info("function type changed to new deployment, creating resources",
|
||||
zap.Any("old_function", oldFn.Metadata),
|
||||
zap.Any("new_function", newFn.Metadata))
|
||||
_, err := deploy.createFunction(newFn, true)
|
||||
if err != nil {
|
||||
deploy.updateStatus(oldFn, err, "error changing the function's type to newdeploy")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
deployChanged := false
|
||||
|
||||
if oldFn.Spec.InvokeStrategy != newFn.Spec.InvokeStrategy {
|
||||
|
||||
// to support backward compatibility, if the function was created in default ns, we fall back to creating the
|
||||
// deployment of the function in fission-function ns, so cleaning up resources there
|
||||
ns := deploy.namespace
|
||||
if newFn.Metadata.Namespace != metav1.NamespaceDefault {
|
||||
ns = newFn.Metadata.Namespace
|
||||
}
|
||||
|
||||
fsvc, err := deploy.fsCache.GetByFunctionUID(newFn.Metadata.UID)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "error updating function due to unable to find function service cache: %v", oldFn)
|
||||
return err
|
||||
}
|
||||
|
||||
hpa, err := deploy.getHpa(ns, fsvc.Name)
|
||||
if err != nil {
|
||||
deploy.updateStatus(oldFn, err, "error getting HPA while updating function")
|
||||
return err
|
||||
}
|
||||
|
||||
hpaChanged := false
|
||||
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale {
|
||||
replicas := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
|
||||
hpa.Spec.MinReplicas = &replicas
|
||||
hpaChanged = true
|
||||
}
|
||||
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale {
|
||||
hpa.Spec.MaxReplicas = int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale)
|
||||
hpaChanged = true
|
||||
}
|
||||
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent != oldFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent {
|
||||
targetCpupercent := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent)
|
||||
hpa.Spec.TargetCPUUtilizationPercentage = &targetCpupercent
|
||||
hpaChanged = true
|
||||
}
|
||||
|
||||
if hpaChanged {
|
||||
err := deploy.updateHpa(hpa)
|
||||
if err != nil {
|
||||
deploy.updateStatus(oldFn, err, "error updating HPA while updating function")
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if oldFn.Spec.Environment != newFn.Spec.Environment ||
|
||||
oldFn.Spec.Package.PackageRef != newFn.Spec.Package.PackageRef ||
|
||||
oldFn.Spec.Package.FunctionName != newFn.Spec.Package.FunctionName {
|
||||
deployChanged = true
|
||||
}
|
||||
|
||||
// If length of slice has changed then no need to check individual elements
|
||||
if len(oldFn.Spec.Secrets) != len(newFn.Spec.Secrets) {
|
||||
deployChanged = true
|
||||
} else {
|
||||
for i, newSecret := range newFn.Spec.Secrets {
|
||||
if newSecret != oldFn.Spec.Secrets[i] {
|
||||
deployChanged = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(oldFn.Spec.ConfigMaps) != len(newFn.Spec.ConfigMaps) {
|
||||
deployChanged = true
|
||||
} else {
|
||||
for i, newConfig := range newFn.Spec.ConfigMaps {
|
||||
if newConfig != oldFn.Spec.ConfigMaps[i] {
|
||||
deployChanged = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if deployChanged == true {
|
||||
env, err := deploy.fissionClient.Environments(newFn.Spec.Environment.Namespace).
|
||||
Get(newFn.Spec.Environment.Name)
|
||||
if err != nil {
|
||||
deploy.updateStatus(oldFn, err, "failed to get environment while updating function")
|
||||
return err
|
||||
}
|
||||
return deploy.updateFuncDeployment(newFn, env)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) updateFuncDeployment(fn *fv1.Function, env *fv1.Environment) error {
|
||||
|
||||
fsvc, err := deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "error updating function due to unable to find function service cache: %v", fn)
|
||||
return err
|
||||
}
|
||||
fnObjName := fsvc.Name
|
||||
|
||||
deployLabels := deploy.getDeployLabels(fn, env)
|
||||
deploy.logger.Info("updating deployment due to function/environment update", zap.String("deployment", fnObjName), zap.Any("function", fn.Metadata.Name))
|
||||
|
||||
newDeployment, err := deploy.getDeploymentSpec(fn, env, fnObjName, deployLabels)
|
||||
if err != nil {
|
||||
deploy.updateStatus(fn, err, "failed to get new deployment spec while updating function")
|
||||
return err
|
||||
}
|
||||
|
||||
// to support backward compatibility, if the function was created in default ns, we fall back to creating the
|
||||
// deployment of the function in fission-function ns
|
||||
ns := deploy.namespace
|
||||
if fn.Metadata.Namespace != metav1.NamespaceDefault {
|
||||
ns = fn.Metadata.Namespace
|
||||
}
|
||||
|
||||
err = deploy.updateDeployment(newDeployment, ns)
|
||||
if err != nil {
|
||||
deploy.updateStatus(fn, err, "failed to update deployment while updating function")
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) fnDelete(fn *fv1.Function) error {
|
||||
var multierr *multierror.Error
|
||||
|
||||
// GetByFunction uses resource version as part of cache key, however,
|
||||
// the resource version in function metadata will be changed when a function
|
||||
// is deleted and cause newdeploy backend fails to delete the entry.
|
||||
// Use GetByFunctionUID instead of GetByFunction here to find correct
|
||||
// fsvc entry.
|
||||
fsvc, err := deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, fmt.Sprintf("fsvc not found in cache: %v", fn.Metadata))
|
||||
return err
|
||||
}
|
||||
|
||||
objName := fsvc.Name
|
||||
|
||||
_, err = deploy.fsCache.DeleteOld(fsvc, time.Second*0)
|
||||
if err != nil {
|
||||
multierr = multierror.Append(multierr,
|
||||
errors.Wrap(err, fmt.Sprintf("error deleting the function from cache")))
|
||||
}
|
||||
|
||||
// to support backward compatibility, if the function was created in default ns, we fall back to creating the
|
||||
// deployment of the function in fission-function ns, so cleaning up resources there
|
||||
ns := deploy.namespace
|
||||
if fn.Metadata.Namespace != metav1.NamespaceDefault {
|
||||
ns = fn.Metadata.Namespace
|
||||
}
|
||||
|
||||
err = deploy.cleanupNewdeploy(ns, objName)
|
||||
multierr = multierror.Append(multierr, err)
|
||||
|
||||
return multierr.ErrorOrNil()
|
||||
}
|
||||
|
||||
// getObjName returns a unique name for kubernetes objects of function
|
||||
func (deploy *NewDeploy) getObjName(fn *fv1.Function) string {
|
||||
return strings.ToLower(fmt.Sprintf("newdeploy-%v-%v-%v", fn.Metadata.Name, fn.Metadata.Namespace, uniuri.NewLen(8)))
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getDeployLabels(fn *fv1.Function, env *fv1.Environment) map[string]string {
|
||||
return map[string]string{
|
||||
types.EXECUTOR_INSTANCEID_LABEL: deploy.instanceID,
|
||||
types.EXECUTOR_TYPE: fv1.ExecutorTypeNewdeploy,
|
||||
types.ENVIRONMENT_NAME: env.Metadata.Name,
|
||||
types.ENVIRONMENT_NAMESPACE: env.Metadata.Namespace,
|
||||
types.ENVIRONMENT_UID: string(env.Metadata.UID),
|
||||
types.FUNCTION_NAME: fn.Metadata.Name,
|
||||
types.FUNCTION_NAMESPACE: fn.Metadata.Namespace,
|
||||
types.FUNCTION_UID: string(fn.Metadata.UID),
|
||||
}
|
||||
}
|
||||
|
||||
// updateKubeObjRefRV update the resource version of kubeObjectRef with
|
||||
// given kind and return error if failed to find the reference.
|
||||
func (deploy *NewDeploy) updateKubeObjRefRV(fsvc *fscache.FuncSvc, objKind string, rv string) error {
|
||||
kubeObjs := fsvc.KubernetesObjects
|
||||
for i, obj := range kubeObjs {
|
||||
if obj.Kind == objKind {
|
||||
kubeObjs[i].ResourceVersion = rv
|
||||
return nil
|
||||
}
|
||||
}
|
||||
fsvc.KubernetesObjects = kubeObjs
|
||||
return fmt.Errorf("error finding kubernetes object reference with kind: %v", objKind)
|
||||
}
|
||||
|
||||
// updateStatus is a function which updates status of update.
|
||||
// Current implementation only logs messages, in future it will update function status
|
||||
func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message string) {
|
||||
deploy.logger.Info("function status update", zap.Error(err), zap.Any("function", fn), zap.String("message", message))
|
||||
}
|
||||
|
||||
// IsValid does a get on the service address to ensure it's a valid service, then
|
||||
// scale deployment to 1 replica if there are no available replicas for function.
|
||||
// Return true if no error occurs, return false otherwise.
|
||||
func (deploy *NewDeploy) IsValid(fsvc *fscache.FuncSvc) bool {
|
||||
service := strings.Split(fsvc.Address, ".")
|
||||
if len(service) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
_, err := deploy.kubernetesClient.CoreV1().Services(service[1]).Get(service[0], metav1.GetOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Error("error validating function service address", zap.String("function", fsvc.Function.Name), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
deployObj := getDeploymentObj(fsvc.KubernetesObjects)
|
||||
if deployObj == nil {
|
||||
deploy.logger.Error("deployment obj for function does not exist", zap.String("function", fsvc.Function.Name))
|
||||
return false
|
||||
}
|
||||
|
||||
currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
|
||||
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
return false
|
||||
}
|
||||
|
||||
// return directly when available replicas > 0
|
||||
if currentDeploy.Status.AvailableReplicas > 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// idleObjectReaper reaps objects after certain idle time
|
||||
func (deploy *NewDeploy) idleObjectReaper() {
|
||||
|
||||
pollSleep := time.Duration(deploy.idlePodReapTime)
|
||||
for {
|
||||
time.Sleep(pollSleep)
|
||||
|
||||
envs, err := deploy.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Fatal("failed to get environment list", zap.Error(err))
|
||||
}
|
||||
|
||||
envList := make(map[k8sTypes.UID]struct{})
|
||||
for _, env := range envs.Items {
|
||||
envList[env.Metadata.UID] = struct{}{}
|
||||
}
|
||||
|
||||
funcSvcs, err := deploy.fsCache.ListOld(deploy.idlePodReapTime)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error reaping idle pods", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
for _, fsvc := range funcSvcs {
|
||||
if fsvc.Executor != fscache.NEWDEPLOY {
|
||||
continue
|
||||
}
|
||||
|
||||
// For function with the environment that no longer exists, executor
|
||||
// scales down the deployment as usual and prints log to notify user.
|
||||
if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
|
||||
deploy.logger.Error("function environment no longer exists",
|
||||
zap.String("environment", fsvc.Environment.Metadata.Name),
|
||||
zap.String("function", fsvc.Name))
|
||||
}
|
||||
|
||||
fn, err := deploy.fissionClient.Functions(fsvc.Function.Namespace).Get(fsvc.Function.Name)
|
||||
if err != nil {
|
||||
// Newdeploy manager handles the function delete event and clean cache/kubeobjs itself,
|
||||
// so we ignore the not found error for functions with newdeploy executor type here.
|
||||
if k8sErrs.IsNotFound(err) && fsvc.Executor == fscache.NEWDEPLOY {
|
||||
continue
|
||||
}
|
||||
deploy.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
continue
|
||||
}
|
||||
|
||||
deployObj := getDeploymentObj(fsvc.KubernetesObjects)
|
||||
if deployObj == nil {
|
||||
deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
continue
|
||||
}
|
||||
|
||||
currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
|
||||
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
continue
|
||||
}
|
||||
|
||||
minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
|
||||
|
||||
// do nothing if the current replicas is already lower than minScale
|
||||
if *currentDeploy.Spec.Replicas <= minScale {
|
||||
continue
|
||||
}
|
||||
|
||||
err = deploy.scaleDeployment(deployObj.Namespace, deployObj.Name, minScale)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getDeploymentObj(kubeobjs []apiv1.ObjectReference) *apiv1.ObjectReference {
|
||||
for _, kubeobj := range kubeobjs {
|
||||
switch strings.ToLower(kubeobj.Kind) {
|
||||
case "deployment":
|
||||
return &kubeobj
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) scaleDeployment(deplNS string, deplName string, replicas int32) error {
|
||||
deploy.logger.Info("scaling deployment",
|
||||
zap.String("deployment", deplName),
|
||||
zap.String("namespace", deplNS),
|
||||
zap.Int32("replicas", replicas))
|
||||
_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deplNS).UpdateScale(deplName, &v1beta1.Scale{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: deplName,
|
||||
Namespace: deplNS,
|
||||
},
|
||||
Spec: v1beta1.ScaleSpec{
|
||||
Replicas: replicas,
|
||||
},
|
||||
})
|
||||
return err
|
||||
}
|
||||
Reference in New Issue
Block a user