Move packages to proejct/pkg to follow go project folder structure convention (#1190)
This commit is contained in:
Ta-Ching Chen
@@ -0,0 +1,150 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
"github.com/gorilla/mux"
|
||||
"go.opencensus.io/plugin/ochttp"
|
||||
"go.uber.org/zap"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
ferror "github.com/fission/fission/pkg/error"
|
||||
)
|
||||
|
||||
func (executor *Executor) getServiceForFunctionApi(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to read request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
// get function metadata
|
||||
m := metav1.ObjectMeta{}
|
||||
err = json.Unmarshal(body, &m)
|
||||
if err != nil {
|
||||
http.Error(w, "Failed to parse request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
serviceName, err := executor.getServiceForFunction(r.Context(), &m)
|
||||
if err != nil {
|
||||
code, msg := ferror.GetHTTPError(err)
|
||||
executor.logger.Error("error getting service for function",
|
||||
zap.Error(err),
|
||||
zap.String("function", m.Name),
|
||||
zap.String("fission_http_error", msg))
|
||||
http.Error(w, msg, code)
|
||||
return
|
||||
}
|
||||
|
||||
w.Write([]byte(serviceName))
|
||||
}
|
||||
|
||||
// getServiceForFunction first checks if this function's service is cached, if yes, it validates the address.
|
||||
// if it's a valid address, just returns it.
|
||||
// else, invalidates its cache entry and makes a new request to create a service for this function and finally responds
|
||||
// with new address or error.
|
||||
//
|
||||
// checking for the validity of the address causes a little more over-head than desired. but, it ensures that
|
||||
// stale addresses are not returned to the router.
|
||||
// To make it optimal, plan is to add an eager cache invalidator function that watches for pod deletion events and
|
||||
// invalidates the cache entry if the pod address was cached.
|
||||
func (executor *Executor) getServiceForFunction(ctx context.Context, m *metav1.ObjectMeta) (string, error) {
|
||||
// Check function -> svc cache
|
||||
executor.logger.Info("checking for cached function service",
|
||||
zap.String("function_name", m.Name),
|
||||
zap.String("function_namespace", m.Namespace))
|
||||
fsvc, err := executor.fsCache.GetByFunction(m)
|
||||
if err == nil {
|
||||
if executor.isValidAddress(fsvc) {
|
||||
// Cached, return svc address
|
||||
return fsvc.Address, nil
|
||||
} else {
|
||||
executor.logger.Info("deleting cache entry for invalid address",
|
||||
zap.String("function_name", m.Name),
|
||||
zap.String("function_namespace", m.Namespace),
|
||||
zap.String("address", fsvc.Address))
|
||||
executor.fsCache.DeleteEntry(fsvc)
|
||||
}
|
||||
}
|
||||
|
||||
respChan := make(chan *createFuncServiceResponse)
|
||||
executor.requestChan <- &createFuncServiceRequest{
|
||||
ctx: ctx,
|
||||
funcMeta: m,
|
||||
respChan: respChan,
|
||||
}
|
||||
resp := <-respChan
|
||||
if resp.err != nil {
|
||||
return "", resp.err
|
||||
}
|
||||
return resp.funcSvc.Address, resp.err
|
||||
}
|
||||
|
||||
// find funcSvc and update its atime
|
||||
func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
executor.logger.Error("failed to read tap service request", zap.Error(err))
|
||||
http.Error(w, "Failed to read request", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
svcName := string(body)
|
||||
svcHost := strings.TrimPrefix(svcName, "http://")
|
||||
|
||||
err = executor.fsCache.TouchByAddress(svcHost)
|
||||
if err != nil {
|
||||
executor.logger.Error("error tapping function service",
|
||||
zap.Error(err),
|
||||
zap.String("service", svcName),
|
||||
zap.String("host", svcHost))
|
||||
http.Error(w, "Not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (executor *Executor) healthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (executor *Executor) Serve(port int) {
|
||||
r := mux.NewRouter()
|
||||
r.HandleFunc("/v2/getServiceForFunction", executor.getServiceForFunctionApi).Methods("POST")
|
||||
r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST")
|
||||
r.HandleFunc("/healthz", executor.healthHandler).Methods("GET")
|
||||
address := fmt.Sprintf(":%v", port)
|
||||
executor.logger.Info("starting executor", zap.Int("port", port))
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
executor.ndm.Run(ctx)
|
||||
executor.gpm.Run(ctx)
|
||||
r.Use(utils.LoggingMiddleware(executor.logger))
|
||||
err := http.ListenAndServe(address, &ochttp.Handler{
|
||||
Handler: r,
|
||||
// Propagation: &b3.HTTPFormat{},
|
||||
})
|
||||
executor.logger.Fatal("done listening", zap.Error(err))
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package client
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"go.opencensus.io/plugin/ochttp"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/net/context/ctxhttp"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
ferror "github.com/fission/fission/pkg/error"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
logger *zap.Logger
|
||||
executorUrl string
|
||||
tappedByUrl map[string]bool
|
||||
requestChan chan string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func MakeClient(logger *zap.Logger, executorUrl string) *Client {
|
||||
c := &Client{
|
||||
logger: logger.Named("executor_client"),
|
||||
executorUrl: strings.TrimSuffix(executorUrl, "/"),
|
||||
tappedByUrl: make(map[string]bool),
|
||||
requestChan: make(chan string),
|
||||
httpClient: &http.Client{
|
||||
Transport: &ochttp.Transport{},
|
||||
},
|
||||
}
|
||||
go c.service()
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Client) GetServiceForFunction(ctx context.Context, metadata *metav1.ObjectMeta) (string, error) {
|
||||
executorUrl := c.executorUrl + "/v2/getServiceForFunction"
|
||||
|
||||
body, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "could not marshal request body for getting service for function")
|
||||
}
|
||||
|
||||
resp, err := ctxhttp.Post(ctx, c.httpClient, executorUrl, "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "error posting to getting service for function")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
return "", ferror.MakeErrorFromHTTP(resp)
|
||||
}
|
||||
|
||||
svcName, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "error reading response body from getting service for function")
|
||||
}
|
||||
|
||||
return string(svcName), nil
|
||||
}
|
||||
|
||||
func (c *Client) service() {
|
||||
ticker := time.NewTicker(time.Second * 5)
|
||||
for {
|
||||
select {
|
||||
case serviceUrl := <-c.requestChan:
|
||||
c.tappedByUrl[serviceUrl] = true
|
||||
case <-ticker.C:
|
||||
urls := c.tappedByUrl
|
||||
c.tappedByUrl = make(map[string]bool)
|
||||
if len(urls) > 0 {
|
||||
go func() {
|
||||
for u := range urls {
|
||||
err := c._tapService(u)
|
||||
if err != nil {
|
||||
c.logger.Error("error tapping function service address", zap.Error(err), zap.String("address", u))
|
||||
}
|
||||
}
|
||||
c.logger.Info("tapped services in batch", zap.Int("service_count", len(urls)))
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) TapService(serviceUrl *url.URL) {
|
||||
c.requestChan <- serviceUrl.String()
|
||||
}
|
||||
|
||||
func (c *Client) _tapService(serviceUrlStr string) error {
|
||||
executorUrl := c.executorUrl + "/v2/tapService"
|
||||
|
||||
resp, err := http.Post(executorUrl, "application/octet-stream", bytes.NewReader([]byte(serviceUrlStr)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return ferror.MakeErrorFromHTTP(resp)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dchest/uniuri"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.uber.org/zap"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
"github.com/fission/fission/pkg/executor/fscache"
|
||||
"github.com/fission/fission/pkg/executor/newdeploy"
|
||||
"github.com/fission/fission/pkg/executor/poolmgr"
|
||||
"github.com/fission/fission/pkg/executor/reaper"
|
||||
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
)
|
||||
|
||||
type (
|
||||
Executor struct {
|
||||
logger *zap.Logger
|
||||
|
||||
gpm *poolmgr.GenericPoolManager
|
||||
ndm *newdeploy.NewDeploy
|
||||
|
||||
fissionClient *crd.FissionClient
|
||||
fsCache *fscache.FunctionServiceCache
|
||||
|
||||
requestChan chan *createFuncServiceRequest
|
||||
fsCreateWg map[string]*sync.WaitGroup
|
||||
}
|
||||
createFuncServiceRequest struct {
|
||||
ctx context.Context
|
||||
funcMeta *metav1.ObjectMeta
|
||||
respChan chan *createFuncServiceResponse
|
||||
}
|
||||
|
||||
createFuncServiceResponse struct {
|
||||
funcSvc *fscache.FuncSvc
|
||||
err error
|
||||
}
|
||||
)
|
||||
|
||||
func MakeExecutor(logger *zap.Logger, gpm *poolmgr.GenericPoolManager, ndm *newdeploy.NewDeploy, fissionClient *crd.FissionClient, fsCache *fscache.FunctionServiceCache) *Executor {
|
||||
executor := &Executor{
|
||||
logger: logger.Named("executor"),
|
||||
gpm: gpm,
|
||||
ndm: ndm,
|
||||
fissionClient: fissionClient,
|
||||
fsCache: fsCache,
|
||||
|
||||
requestChan: make(chan *createFuncServiceRequest),
|
||||
fsCreateWg: make(map[string]*sync.WaitGroup),
|
||||
}
|
||||
go executor.serveCreateFuncServices()
|
||||
|
||||
return executor
|
||||
}
|
||||
|
||||
// All non-cached function service requests go through this goroutine
|
||||
// serially. It parallelizes requests for different functions, and
|
||||
// ensures that for a given function, only one request causes a pod to
|
||||
// get specialized. In other words, it ensures that when there's an
|
||||
// ongoing request for a certain function, all other requests wait for
|
||||
// that request to complete.
|
||||
func (executor *Executor) serveCreateFuncServices() {
|
||||
for {
|
||||
req := <-executor.requestChan
|
||||
m := req.funcMeta
|
||||
|
||||
// Cache miss -- is this first one to request the func?
|
||||
wg, found := executor.fsCreateWg[crd.CacheKey(m)]
|
||||
if !found {
|
||||
// create a waitgroup for other requests for
|
||||
// the same function to wait on
|
||||
wg := &sync.WaitGroup{}
|
||||
wg.Add(1)
|
||||
executor.fsCreateWg[crd.CacheKey(m)] = wg
|
||||
|
||||
// launch a goroutine for each request, to parallelize
|
||||
// the specialization of different functions
|
||||
go func() {
|
||||
fsvc, err := executor.createServiceForFunction(req.ctx, m)
|
||||
req.respChan <- &createFuncServiceResponse{
|
||||
funcSvc: fsvc,
|
||||
err: err,
|
||||
}
|
||||
delete(executor.fsCreateWg, crd.CacheKey(m))
|
||||
wg.Done()
|
||||
}()
|
||||
} else {
|
||||
// There's an existing request for this function, wait for it to finish
|
||||
go func() {
|
||||
executor.logger.Info("waiting for concurrent request for the same function",
|
||||
zap.Any("function", m))
|
||||
wg.Wait()
|
||||
|
||||
// get the function service from the cache
|
||||
fsvc, err := executor.fsCache.GetByFunction(m)
|
||||
|
||||
// fsCache return error when the entry does not exist/expire.
|
||||
// It normally happened if there are multiple requests are
|
||||
// waiting for the same function and executor failed to cre-
|
||||
// ate service for function.
|
||||
err = errors.Wrapf(err, "error getting service for function",
|
||||
zap.String("function_name", m.Name),
|
||||
zap.String("function_namespace", m.Namespace))
|
||||
req.respChan <- &createFuncServiceResponse{
|
||||
funcSvc: fsvc,
|
||||
err: err,
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (executor *Executor) getFunctionExecutorType(meta *metav1.ObjectMeta) (fv1.ExecutorType, error) {
|
||||
fn, err := executor.fissionClient.Functions(meta.Namespace).Get(meta.Name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType, nil
|
||||
}
|
||||
|
||||
func (executor *Executor) createServiceForFunction(ctx context.Context, meta *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
|
||||
executor.logger.Info("no cached function service found, creating one",
|
||||
zap.String("function_name", meta.Name),
|
||||
zap.String("function_namespace", meta.Namespace))
|
||||
|
||||
executorType, err := executor.getFunctionExecutorType(meta)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var fsvc *fscache.FuncSvc
|
||||
var fsvcErr error
|
||||
|
||||
switch executorType {
|
||||
case fv1.ExecutorTypeNewdeploy:
|
||||
fsvc, fsvcErr = executor.ndm.GetFuncSvc(ctx, meta)
|
||||
default:
|
||||
fsvc, fsvcErr = executor.gpm.GetFuncSvc(ctx, meta)
|
||||
}
|
||||
|
||||
if fsvcErr != nil {
|
||||
e := "error creating service for function"
|
||||
executor.logger.Error(e,
|
||||
zap.Error(fsvcErr),
|
||||
zap.String("function_name", meta.Name),
|
||||
zap.String("function_namespace", meta.Namespace))
|
||||
fsvcErr = errors.Wrap(fsvcErr, fmt.Sprintf("[%s] %s", meta.Name, e))
|
||||
} else if fsvc != nil {
|
||||
_, err = executor.fsCache.Add(*fsvc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
executor.fsCache.IncreaseColdStarts(meta.Name, string(meta.UID))
|
||||
|
||||
return fsvc, fsvcErr
|
||||
}
|
||||
|
||||
// isValidAddress invokes isValidService or isValidPod depending on the type of executor
|
||||
func (executor *Executor) isValidAddress(fsvc *fscache.FuncSvc) bool {
|
||||
if fsvc.Executor == fscache.NEWDEPLOY {
|
||||
return executor.ndm.IsValid(fsvc)
|
||||
} else {
|
||||
return executor.gpm.IsValid(fsvc)
|
||||
}
|
||||
}
|
||||
|
||||
func serveMetric(logger *zap.Logger) {
|
||||
// Expose the registered metrics via HTTP.
|
||||
metricAddr := ":8080"
|
||||
http.Handle("/metrics", promhttp.Handler())
|
||||
err := http.ListenAndServe(metricAddr, nil)
|
||||
|
||||
logger.Fatal("done listening on metrics endpoint", zap.Error(err))
|
||||
}
|
||||
|
||||
// StartExecutor Starts executor and the executor components such as Poolmgr,
|
||||
// deploymgr and potential future executor types
|
||||
func StartExecutor(logger *zap.Logger, fissionNamespace string, functionNamespace string, envBuilderNamespace string, port int) error {
|
||||
// setup a signal handler for SIGTERM
|
||||
utils.SetupStackTraceHandler()
|
||||
|
||||
fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
|
||||
|
||||
err = fissionClient.WaitForCRDs()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error waiting for CRDs")
|
||||
}
|
||||
|
||||
fetcherConfig, err := fetcherConfig.MakeFetcherConfig("/userfunc")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Error making fetcher config")
|
||||
}
|
||||
|
||||
restClient := fissionClient.GetCrdClient()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to get kubernetes client")
|
||||
}
|
||||
|
||||
fsCache := fscache.MakeFunctionServiceCache(logger)
|
||||
|
||||
poolID := strings.ToLower(uniuri.NewLen(8))
|
||||
reaper.CleanupOldExecutorObjects(logger, kubernetesClient, poolID)
|
||||
go reaper.CleanupRoleBindings(logger, kubernetesClient, fissionClient, functionNamespace, envBuilderNamespace, time.Minute*30)
|
||||
|
||||
gpm := poolmgr.MakeGenericPoolManager(
|
||||
logger,
|
||||
fissionClient, kubernetesClient,
|
||||
functionNamespace, fetcherConfig, poolID)
|
||||
|
||||
ndm := newdeploy.MakeNewDeploy(
|
||||
logger,
|
||||
fissionClient, kubernetesClient, restClient,
|
||||
functionNamespace, fetcherConfig, poolID)
|
||||
|
||||
api := MakeExecutor(logger, gpm, ndm, fissionClient, fsCache)
|
||||
|
||||
go api.Serve(port)
|
||||
go serveMetric(logger)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
//
|
||||
// This test depends on several env vars:
|
||||
//
|
||||
// KUBECONFIG has to point at a kube config with a cluster. The test
|
||||
// will use the default context from that config. Be careful,
|
||||
// don't point this at your production environment. The test is
|
||||
// skipped if KUBECONFIG is undefined.
|
||||
//
|
||||
// TEST_SPECIALIZE_URL
|
||||
// TEST_FETCHER_URL
|
||||
// These need to point at <node ip>:30001 and <node ip>:30002,
|
||||
// where <node ip> is the address of any node in the test
|
||||
// cluster.
|
||||
//
|
||||
// FETCHER_IMAGE
|
||||
// Optional. Set this to a fetcher image; otherwise uses the
|
||||
// default.
|
||||
//
|
||||
|
||||
// Here's how I run this on my setup, with minikube:
|
||||
// TEST_SPECIALIZE_URL=http://192.168.99.100:30002/specialize TEST_FETCHER_URL=http://192.168.99.100:30001 FETCHER_IMAGE=minikube/fetcher:testing KUBECONFIG=/Users/soam/.kube/config go test -v .
|
||||
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
"github.com/fission/fission/pkg/executor/client"
|
||||
)
|
||||
|
||||
func panicIf(err error) {
|
||||
if err != nil {
|
||||
log.Panicf("Error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// return the number of pods in the given namespace matching the given labels
|
||||
func countPods(kubeClient *kubernetes.Clientset, ns string, labelz map[string]string) int {
|
||||
pods, err := kubeClient.CoreV1().Pods(ns).List(metav1.ListOptions{
|
||||
LabelSelector: labels.Set(labelz).AsSelector().String(),
|
||||
})
|
||||
if err != nil {
|
||||
log.Panicf("Failed to list pods: %v", err)
|
||||
}
|
||||
return len(pods.Items)
|
||||
}
|
||||
|
||||
func createTestNamespace(kubeClient *kubernetes.Clientset, ns string) {
|
||||
_, err := kubeClient.CoreV1().Namespaces().Create(&apiv1.Namespace{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: ns,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Panicf("failed to create ns %v: %v", ns, err)
|
||||
}
|
||||
log.Printf("Created namespace %v", ns)
|
||||
}
|
||||
|
||||
// create a nodeport service
|
||||
func createSvc(kubeClient *kubernetes.Clientset, ns string, name string, targetPort int, nodePort int32, labels map[string]string) *apiv1.Service {
|
||||
svc, err := kubeClient.CoreV1().Services(ns).Create(&apiv1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Spec: apiv1.ServiceSpec{
|
||||
Type: apiv1.ServiceTypeNodePort,
|
||||
Ports: []apiv1.ServicePort{
|
||||
{
|
||||
Protocol: apiv1.ProtocolTCP,
|
||||
Port: 80,
|
||||
TargetPort: intstr.FromInt(targetPort),
|
||||
NodePort: nodePort,
|
||||
},
|
||||
},
|
||||
Selector: labels,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Panicf("Failed to create svc: %v", err)
|
||||
}
|
||||
return svc
|
||||
}
|
||||
|
||||
func httpGet(url string) string {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
log.Panicf("HTTP Get failed: URL %v: %v", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Panicf("HTTP Get failed to read body: URL %v: %v", url, err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
|
||||
func TestExecutor(t *testing.T) {
|
||||
// run in a random namespace so we can have concurrent tests
|
||||
// on a given cluster
|
||||
rand.Seed(time.Now().UTC().UnixNano())
|
||||
testId := rand.Intn(999)
|
||||
fissionNs := fmt.Sprintf("test-%v", testId)
|
||||
functionNs := fmt.Sprintf("test-function-%v", testId)
|
||||
|
||||
// skip test if no cluster available for testing
|
||||
kubeconfig := os.Getenv("KUBECONFIG")
|
||||
if len(kubeconfig) == 0 {
|
||||
t.Skip("Skipping test, no kubernetes cluster")
|
||||
return
|
||||
}
|
||||
|
||||
// connect to k8s
|
||||
// and get CRD client
|
||||
fissionClient, kubeClient, apiExtClient, err := crd.MakeFissionClient()
|
||||
if err != nil {
|
||||
log.Panicf("failed to connect: %v", err)
|
||||
}
|
||||
|
||||
// create the test's namespaces
|
||||
createTestNamespace(kubeClient, fissionNs)
|
||||
defer kubeClient.CoreV1().Namespaces().Delete(fissionNs, nil)
|
||||
|
||||
createTestNamespace(kubeClient, functionNs)
|
||||
defer kubeClient.CoreV1().Namespaces().Delete(functionNs, nil)
|
||||
|
||||
logger, err := zap.NewDevelopment()
|
||||
panicIf(err)
|
||||
|
||||
// make sure CRD types exist on cluster
|
||||
err = crd.EnsureFissionCRDs(logger, apiExtClient)
|
||||
if err != nil {
|
||||
log.Panicf("failed to ensure crds: %v", err)
|
||||
}
|
||||
|
||||
err = fissionClient.WaitForCRDs()
|
||||
if err != nil {
|
||||
log.Panicf("failed to wait crds: %v", err)
|
||||
}
|
||||
|
||||
// create an env on the cluster
|
||||
env, err := fissionClient.Environments(fissionNs).Create(&fv1.Environment{
|
||||
Metadata: metav1.ObjectMeta{
|
||||
Name: "nodejs",
|
||||
Namespace: fissionNs,
|
||||
},
|
||||
Spec: fv1.EnvironmentSpec{
|
||||
Version: 1,
|
||||
Runtime: fv1.Runtime{
|
||||
Image: "fission/node-env",
|
||||
},
|
||||
Builder: fv1.Builder{},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Panicf("failed to create env: %v", err)
|
||||
}
|
||||
|
||||
// create poolmgr
|
||||
port := 9999
|
||||
err = StartExecutor(logger, fissionNs, functionNs, "fission-builder", port)
|
||||
if err != nil {
|
||||
log.Panicf("failed to start poolmgr: %v", err)
|
||||
}
|
||||
|
||||
// connect poolmgr client
|
||||
poolmgrClient := client.MakeClient(logger, fmt.Sprintf("http://localhost:%v", port))
|
||||
|
||||
// Wait for pool to be created (we don't actually need to do
|
||||
// this, since the API should do the right thing in any case).
|
||||
// waitForPool(functionNs, "nodejs")
|
||||
time.Sleep(6 * time.Second)
|
||||
|
||||
envRef := fv1.EnvironmentReference{
|
||||
Namespace: env.Metadata.Namespace,
|
||||
Name: env.Metadata.Name,
|
||||
}
|
||||
|
||||
deployment := fv1.Archive{
|
||||
Type: fv1.ArchiveTypeLiteral,
|
||||
Literal: []byte(`module.exports = async function(context) { return { status: 200, body: "Hello, world!\n" }; }`),
|
||||
}
|
||||
|
||||
// create a package
|
||||
p := &fv1.Package{
|
||||
Metadata: metav1.ObjectMeta{
|
||||
Name: "hello",
|
||||
Namespace: fissionNs,
|
||||
},
|
||||
Spec: fv1.PackageSpec{
|
||||
Environment: envRef,
|
||||
Deployment: deployment,
|
||||
},
|
||||
}
|
||||
p, err = fissionClient.Packages(fissionNs).Create(p)
|
||||
if err != nil {
|
||||
log.Panicf("failed to create package: %v", err)
|
||||
}
|
||||
|
||||
// create a function
|
||||
f := &fv1.Function{
|
||||
Metadata: metav1.ObjectMeta{
|
||||
Name: "hello",
|
||||
Namespace: fissionNs,
|
||||
},
|
||||
Spec: fv1.FunctionSpec{
|
||||
Environment: envRef,
|
||||
Package: fv1.FunctionPackageRef{
|
||||
PackageRef: fv1.PackageRef{
|
||||
Namespace: p.Metadata.Namespace,
|
||||
Name: p.Metadata.Name,
|
||||
ResourceVersion: p.Metadata.ResourceVersion,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = fissionClient.Functions(fissionNs).Create(f)
|
||||
if err != nil {
|
||||
log.Panicf("failed to create function: %v", err)
|
||||
}
|
||||
|
||||
// create a service to call fetcher and the env container
|
||||
labels := map[string]string{"functionName": f.Metadata.Name}
|
||||
var fetcherPort int32 = 30001
|
||||
fetcherSvc := createSvc(kubeClient, functionNs, fmt.Sprintf("%v-%v", f.Metadata.Name, "fetcher"), 8000, fetcherPort, labels)
|
||||
defer kubeClient.CoreV1().Services(functionNs).Delete(fetcherSvc.ObjectMeta.Name, nil)
|
||||
|
||||
var funcSvcPort int32 = 30002
|
||||
functionSvc := createSvc(kubeClient, functionNs, f.Metadata.Name, 8888, funcSvcPort, labels)
|
||||
defer kubeClient.CoreV1().Services(functionNs).Delete(functionSvc.ObjectMeta.Name, nil)
|
||||
|
||||
// the main test: get a service for a given function
|
||||
t1 := time.Now()
|
||||
svc, err := poolmgrClient.GetServiceForFunction(context.Background(), &f.Metadata)
|
||||
if err != nil {
|
||||
log.Panicf("failed to get func svc: %v", err)
|
||||
}
|
||||
log.Printf("svc for function created at: %v (in %v)", svc, time.Since(t1))
|
||||
|
||||
// ensure that a pod with the label functionName=f.Metadata.Name exists
|
||||
podCount := countPods(kubeClient, functionNs, map[string]string{"functionName": f.Metadata.Name})
|
||||
if podCount != 1 {
|
||||
log.Panicf("expected 1 function pod, found %v", podCount)
|
||||
}
|
||||
|
||||
// call the service to ensure it works
|
||||
|
||||
// wait for a bit
|
||||
|
||||
// tap service to simulate calling it again
|
||||
|
||||
// make sure the same pod is still there
|
||||
|
||||
// wait for idleTimeout to ensure the pod is removed
|
||||
|
||||
// remove env
|
||||
|
||||
// wait for pool to be destroyed
|
||||
|
||||
// that's it
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package fscache
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/cache"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
ferror "github.com/fission/fission/pkg/error"
|
||||
)
|
||||
|
||||
type fscRequestType int
|
||||
type executorType int
|
||||
|
||||
const (
|
||||
TOUCH fscRequestType = iota
|
||||
LISTOLD
|
||||
LOG
|
||||
)
|
||||
|
||||
const (
|
||||
POOLMGR executorType = iota
|
||||
NEWDEPLOY
|
||||
)
|
||||
|
||||
type (
|
||||
FuncSvc struct {
|
||||
Name string // Name of object
|
||||
Function *metav1.ObjectMeta // function this pod/service is for
|
||||
Environment *fv1.Environment // function's environment
|
||||
Address string // Host:Port or IP:Port that the function's service can be reached at.
|
||||
KubernetesObjects []apiv1.ObjectReference // Kubernetes Objects (within the function namespace)
|
||||
Executor executorType
|
||||
|
||||
Ctime time.Time
|
||||
Atime time.Time
|
||||
}
|
||||
|
||||
FunctionServiceCache struct {
|
||||
logger *zap.Logger
|
||||
byFunction *cache.Cache // function-key -> funcSvc : map[string]*funcSvc
|
||||
byAddress *cache.Cache // address -> function : map[string]metav1.ObjectMeta
|
||||
byFunctionUID *cache.Cache // function uid -> function : map[string]metav1.ObjectMeta
|
||||
|
||||
requestChannel chan *fscRequest
|
||||
}
|
||||
fscRequest struct {
|
||||
requestType fscRequestType
|
||||
address string
|
||||
kubernetesObjects []apiv1.ObjectReference
|
||||
age time.Duration
|
||||
responseChannel chan *fscResponse
|
||||
}
|
||||
fscResponse struct {
|
||||
objects []*FuncSvc
|
||||
deleted bool
|
||||
error
|
||||
}
|
||||
)
|
||||
|
||||
func IsNotFoundError(err error) bool {
|
||||
if fe, ok := err.(ferror.Error); ok {
|
||||
return fe.Code == ferror.ErrorNotFound
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func IsNameExistError(err error) bool {
|
||||
if fe, ok := err.(ferror.Error); ok {
|
||||
return fe.Code == ferror.ErrorNameExists
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func MakeFunctionServiceCache(logger *zap.Logger) *FunctionServiceCache {
|
||||
fsc := &FunctionServiceCache{
|
||||
logger: logger.Named("function_service_cache"),
|
||||
byFunction: cache.MakeCache(0, 0),
|
||||
byAddress: cache.MakeCache(0, 0),
|
||||
byFunctionUID: cache.MakeCache(0, 0),
|
||||
requestChannel: make(chan *fscRequest),
|
||||
}
|
||||
go fsc.service()
|
||||
return fsc
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) service() {
|
||||
for {
|
||||
req := <-fsc.requestChannel
|
||||
resp := &fscResponse{}
|
||||
switch req.requestType {
|
||||
case TOUCH:
|
||||
// update atime for this function svc
|
||||
resp.error = fsc._touchByAddress(req.address)
|
||||
case LISTOLD:
|
||||
// get svcs idle for > req.age
|
||||
fscs := fsc.byFunction.Copy()
|
||||
funcObjects := make([]*FuncSvc, 0)
|
||||
for _, funcSvc := range fscs {
|
||||
fsvc := funcSvc.(*FuncSvc)
|
||||
if time.Since(fsvc.Atime) > req.age {
|
||||
funcObjects = append(funcObjects, fsvc)
|
||||
}
|
||||
}
|
||||
resp.objects = funcObjects
|
||||
case LOG:
|
||||
fsc.logger.Info("dumping function service cache")
|
||||
funcCopy := fsc.byFunction.Copy()
|
||||
info := []string{}
|
||||
for key, fsvcI := range funcCopy {
|
||||
fsvc := fsvcI.(*FuncSvc)
|
||||
for _, kubeObj := range fsvc.KubernetesObjects {
|
||||
info = append(info, fmt.Sprintf("%v\t%v\t%v", key, kubeObj.Kind, kubeObj.Name))
|
||||
}
|
||||
}
|
||||
fsc.logger.Info("function service cache", zap.Int("item_count", len(funcCopy)), zap.Strings("cache", info))
|
||||
}
|
||||
req.responseChannel <- resp
|
||||
}
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) GetByFunction(m *metav1.ObjectMeta) (*FuncSvc, error) {
|
||||
key := crd.CacheKey(m)
|
||||
|
||||
fsvcI, err := fsc.byFunction.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// update atime
|
||||
fsvc := fsvcI.(*FuncSvc)
|
||||
fsvc.Atime = time.Now()
|
||||
|
||||
fsvcCopy := *fsvc
|
||||
return &fsvcCopy, nil
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) GetByFunctionUID(uid types.UID) (*FuncSvc, error) {
|
||||
mI, err := fsc.byFunctionUID.Get(uid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m := mI.(metav1.ObjectMeta)
|
||||
|
||||
fsvcI, err := fsc.byFunction.Get(crd.CacheKey(&m))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// update atime
|
||||
fsvc := fsvcI.(*FuncSvc)
|
||||
fsvc.Atime = time.Now()
|
||||
|
||||
fsvcCopy := *fsvc
|
||||
return &fsvcCopy, nil
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) Add(fsvc FuncSvc) (*FuncSvc, error) {
|
||||
err, existing := fsc.byFunction.Set(crd.CacheKey(fsvc.Function), &fsvc)
|
||||
if err != nil {
|
||||
if IsNameExistError(err) {
|
||||
f := existing.(*FuncSvc)
|
||||
err2 := fsc.TouchByAddress(f.Address)
|
||||
if err2 != nil {
|
||||
return nil, err2
|
||||
}
|
||||
fCopy := *f
|
||||
return &fCopy, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
now := time.Now()
|
||||
fsvc.Ctime = now
|
||||
fsvc.Atime = now
|
||||
|
||||
// Add to byAddress cache. Ignore NameExists errors
|
||||
// because of multiple-specialization. See issue #331.
|
||||
err, _ = fsc.byAddress.Set(fsvc.Address, *fsvc.Function)
|
||||
if err != nil {
|
||||
if IsNameExistError(err) {
|
||||
err = nil
|
||||
} else {
|
||||
err = errors.Wrap(err, "error caching fsvc")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add to byFunctionUID cache. Ignore NameExists errors
|
||||
// because of multiple-specialization. See issue #331.
|
||||
err, _ = fsc.byFunctionUID.Set(fsvc.Function.UID, *fsvc.Function)
|
||||
if err != nil {
|
||||
if IsNameExistError(err) {
|
||||
err = nil
|
||||
} else {
|
||||
err = errors.Wrap(err, "error caching fsvc by function uid")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fsc.setFuncAlive(fsvc.Function.Name, string(fsvc.Function.UID), true)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) TouchByAddress(address string) error {
|
||||
responseChannel := make(chan *fscResponse)
|
||||
fsc.requestChannel <- &fscRequest{
|
||||
requestType: TOUCH,
|
||||
address: address,
|
||||
responseChannel: responseChannel,
|
||||
}
|
||||
resp := <-responseChannel
|
||||
return resp.error
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) _touchByAddress(address string) error {
|
||||
mI, err := fsc.byAddress.Get(address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m := mI.(metav1.ObjectMeta)
|
||||
fsvcI, err := fsc.byFunction.Get(crd.CacheKey(&m))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fsvc := fsvcI.(*FuncSvc)
|
||||
fsvc.Atime = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) DeleteEntry(fsvc *FuncSvc) {
|
||||
fsc.byFunction.Delete(crd.CacheKey(fsvc.Function))
|
||||
fsc.byAddress.Delete(fsvc.Address)
|
||||
fsc.byFunctionUID.Delete(fsvc.Function.UID)
|
||||
|
||||
fsc.observeFuncRunningTime(fsvc.Function.Name, string(fsvc.Function.UID), fsvc.Atime.Sub(fsvc.Ctime).Seconds())
|
||||
fsc.observeFuncAliveTime(fsvc.Function.Name, string(fsvc.Function.UID), time.Now().Sub(fsvc.Ctime).Seconds())
|
||||
fsc.setFuncAlive(fsvc.Function.Name, string(fsvc.Function.UID), false)
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) DeleteOld(fsvc *FuncSvc, minAge time.Duration) (bool, error) {
|
||||
if time.Since(fsvc.Atime) < minAge {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
fsc.DeleteEntry(fsvc)
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) ListOld(age time.Duration) ([]*FuncSvc, error) {
|
||||
responseChannel := make(chan *fscResponse)
|
||||
fsc.requestChannel <- &fscRequest{
|
||||
requestType: LISTOLD,
|
||||
age: age,
|
||||
responseChannel: responseChannel,
|
||||
}
|
||||
resp := <-responseChannel
|
||||
return resp.objects, resp.error
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) Log() {
|
||||
fsc.logger.Info("--- FunctionService Cache Contents")
|
||||
responseChannel := make(chan *fscResponse)
|
||||
fsc.requestChannel <- &fscRequest{
|
||||
requestType: LOG,
|
||||
responseChannel: responseChannel,
|
||||
}
|
||||
<-responseChannel
|
||||
fsc.logger.Info("--- FunctionService Cache Contents End")
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package fscache
|
||||
|
||||
import (
|
||||
"log"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
)
|
||||
|
||||
func panicIf(err error) {
|
||||
if err != nil {
|
||||
log.Panicf("Error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFunctionServiceCache(t *testing.T) {
|
||||
logger, err := zap.NewDevelopment()
|
||||
panicIf(err)
|
||||
|
||||
fsc := MakeFunctionServiceCache(logger)
|
||||
if fsc == nil {
|
||||
log.Panicf("error creating cache")
|
||||
}
|
||||
|
||||
var fsvc *FuncSvc
|
||||
now := time.Now()
|
||||
|
||||
objects := []apiv1.ObjectReference{
|
||||
{
|
||||
Kind: "pod",
|
||||
Name: "xxx",
|
||||
APIVersion: "v1",
|
||||
Namespace: "fission-function",
|
||||
},
|
||||
{
|
||||
Kind: "pod",
|
||||
Name: "xxx2",
|
||||
APIVersion: "v1",
|
||||
Namespace: "fission-function",
|
||||
},
|
||||
}
|
||||
|
||||
fsvc = &FuncSvc{
|
||||
Function: &metav1.ObjectMeta{
|
||||
Name: "foo",
|
||||
UID: "1212",
|
||||
},
|
||||
Environment: &fv1.Environment{
|
||||
Metadata: metav1.ObjectMeta{
|
||||
Name: "foo-env",
|
||||
UID: "2323",
|
||||
},
|
||||
Spec: fv1.EnvironmentSpec{
|
||||
Version: 1,
|
||||
Runtime: fv1.Runtime{
|
||||
Image: "fission/foo-env",
|
||||
},
|
||||
Builder: fv1.Builder{},
|
||||
},
|
||||
},
|
||||
Address: "xxx",
|
||||
KubernetesObjects: objects,
|
||||
Ctime: now,
|
||||
Atime: now,
|
||||
}
|
||||
_, err = fsc.Add(*fsvc)
|
||||
if err != nil {
|
||||
fsc.Log()
|
||||
log.Panicf("Failed to add fsvc: %v", err)
|
||||
}
|
||||
|
||||
f, err := fsc.GetByFunction(fsvc.Function)
|
||||
if err != nil {
|
||||
fsc.Log()
|
||||
log.Panicf("Failed to get fsvc: %v", err)
|
||||
}
|
||||
f, err = fsc.GetByFunctionUID(fsvc.Function.UID)
|
||||
if err != nil {
|
||||
fsc.Log()
|
||||
log.Panicf("Failed to get fsvc by function uid: %v", err)
|
||||
}
|
||||
fsvc.Atime = f.Atime
|
||||
fsvc.Ctime = f.Ctime
|
||||
if f.Address != fsvc.Address {
|
||||
fsc.Log()
|
||||
log.Panicf("Incorrect fsvc \n(expected: %#v)\n (found: %#v)", fsvc, f)
|
||||
}
|
||||
|
||||
err = fsc.TouchByAddress(fsvc.Address)
|
||||
if err != nil {
|
||||
fsc.Log()
|
||||
log.Panicf("Failed to touch fsvc: %v", err)
|
||||
}
|
||||
|
||||
deleted, err := fsc.DeleteOld(fsvc, 0)
|
||||
if err != nil {
|
||||
fsc.Log()
|
||||
log.Panicf("Failed to delete fsvc: %v", err)
|
||||
}
|
||||
if !deleted {
|
||||
fsc.Log()
|
||||
log.Panicf("Did not delete fsvc")
|
||||
}
|
||||
|
||||
_, err = fsc.GetByFunction(fsvc.Function)
|
||||
if err == nil {
|
||||
fsc.Log()
|
||||
log.Panicf("found fsvc while expecting empty cache: %v", err)
|
||||
}
|
||||
|
||||
_, err = fsc.GetByFunctionUID(fsvc.Function.UID)
|
||||
if err == nil {
|
||||
fsc.Log()
|
||||
log.Panicf("found fsvc by function uid while expecting empty cache: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package fscache
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
var (
|
||||
metricAddr = ":8080"
|
||||
|
||||
// funcname: the function's name
|
||||
// funcuid: the function's version id
|
||||
coldStarts = prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Name: "fission_cold_starts_total",
|
||||
Help: "How many cold starts are made by funcname, funcuid.",
|
||||
},
|
||||
[]string{"funcname", "funcuid"},
|
||||
)
|
||||
funcRunningSummary = prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Name: "fission_func_running_seconds_summary",
|
||||
Help: "The running time (last access - create) in seconds of the function.",
|
||||
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
|
||||
},
|
||||
[]string{"funcname", "funcuid"},
|
||||
)
|
||||
funcAliveSummary = prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Name: "fission_func_alive_seconds_summary",
|
||||
Help: "The alive time in seconds of the function.",
|
||||
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
|
||||
},
|
||||
[]string{"funcname", "funcuid"},
|
||||
)
|
||||
funcIsAlive = prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Name: "fission_func_is_alive",
|
||||
Help: "A binary value indicating is the funcname, funcuid alive",
|
||||
},
|
||||
[]string{"funcname", "funcuid"},
|
||||
)
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Register the function calls counter with Prometheus's default registry.
|
||||
prometheus.MustRegister(coldStarts)
|
||||
prometheus.MustRegister(funcRunningSummary)
|
||||
prometheus.MustRegister(funcAliveSummary)
|
||||
prometheus.MustRegister(funcIsAlive)
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) IncreaseColdStarts(funcname, funcuid string) {
|
||||
coldStarts.WithLabelValues(funcname, funcuid).Inc()
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) observeFuncRunningTime(funcname, funcuid string, running float64) {
|
||||
funcRunningSummary.WithLabelValues(funcname, funcuid).Observe(running)
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) observeFuncAliveTime(funcname, funcuid string, alive float64) {
|
||||
funcAliveSummary.WithLabelValues(funcname, funcuid).Observe(alive)
|
||||
}
|
||||
|
||||
func (fsc *FunctionServiceCache) setFuncAlive(funcname, funcuid string, isAlive bool) {
|
||||
count := 0
|
||||
if isAlive {
|
||||
count = 1
|
||||
}
|
||||
funcIsAlive.WithLabelValues(funcname, funcuid).Set(float64(count))
|
||||
}
|
||||
@@ -0,0 +1,447 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package newdeploy
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/fission/fission/pkg/types"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"go.uber.org/zap"
|
||||
asv1 "k8s.io/api/autoscaling/v1"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
"k8s.io/api/extensions/v1beta1"
|
||||
k8s_err "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/executor/util"
|
||||
)
|
||||
|
||||
const (
|
||||
DeploymentKind = "Deployment"
|
||||
DeploymentVersion = "extensions/v1beta1"
|
||||
)
|
||||
|
||||
func (deploy *NewDeploy) createOrGetDeployment(fn *fv1.Function, env *fv1.Environment,
|
||||
deployName string, deployLabels map[string]string, deployNamespace string, firstcreate bool) (*v1beta1.Deployment, error) {
|
||||
|
||||
minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
|
||||
|
||||
// If it's not the first time creation and minscale is 0 means that all pods for function were recycled,
|
||||
// in such cases we need set minscale to 1 for router to serve requests.
|
||||
if !firstcreate && minScale <= 0 {
|
||||
minScale = 1
|
||||
}
|
||||
|
||||
waitForDeploy := minScale > 0
|
||||
|
||||
existingDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Get(deployName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
if waitForDeploy {
|
||||
err = deploy.scaleDeployment(existingDepl.Namespace, existingDepl.Name, minScale)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error scaling up function deployment", zap.Error(err), zap.String("function", fn.Metadata.Name))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if existingDepl.Status.AvailableReplicas < minScale {
|
||||
existingDepl, err = deploy.waitForDeploy(existingDepl, minScale)
|
||||
}
|
||||
}
|
||||
return existingDepl, err
|
||||
}
|
||||
|
||||
if err != nil && k8s_err.IsNotFound(err) {
|
||||
err := deploy.setupRBACObjs(deployNamespace, fn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
deployment, err := deploy.getDeploymentSpec(fn, env, deployName, deployLabels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
depl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deployNamespace).Create(deployment)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error while creating function deployment",
|
||||
zap.Error(err),
|
||||
zap.String("function", fn.Metadata.Name),
|
||||
zap.String("deployment_name", deployName),
|
||||
zap.String("deployment_namespace", deployNamespace))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if waitForDeploy {
|
||||
depl, err = deploy.waitForDeploy(depl, minScale)
|
||||
}
|
||||
|
||||
return depl, err
|
||||
}
|
||||
|
||||
return nil, err
|
||||
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) setupRBACObjs(deployNamespace string, fn *fv1.Function) error {
|
||||
// create fetcher SA in this ns, if not already created
|
||||
err := deploy.fetcherConfig.SetupServiceAccount(deploy.kubernetesClient, deployNamespace, fn.Metadata)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating fission fetcher service account for function",
|
||||
zap.Error(err),
|
||||
zap.String("service_account_name", types.FissionFetcherSA),
|
||||
zap.String("service_account_namespace", deployNamespace),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
return err
|
||||
}
|
||||
|
||||
// create a cluster role binding for the fetcher SA, if not already created, granting access to do a get on packages in any ns
|
||||
err = utils.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, types.PackageGetterRB, fn.Spec.Package.PackageRef.Namespace, types.PackageGetterCR, types.ClusterRole, types.FissionFetcherSA, deployNamespace)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating role binding for function",
|
||||
zap.Error(err),
|
||||
zap.String("role_binding", types.PackageGetterRB),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
return err
|
||||
}
|
||||
|
||||
// create rolebinding in function namespace for fetcherSA.envNamespace to be able to get secrets and configmaps
|
||||
err = utils.SetupRoleBinding(deploy.logger, deploy.kubernetesClient, types.SecretConfigMapGetterRB, fn.Metadata.Namespace, types.SecretConfigMapGetterCR, types.ClusterRole, types.FissionFetcherSA, deployNamespace)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating role binding for function",
|
||||
zap.Error(err),
|
||||
zap.String("role_binding", types.SecretConfigMapGetterRB),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
return err
|
||||
}
|
||||
|
||||
deploy.logger.Info("set up all RBAC objects for function",
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getDeployment(ns, name string) (*v1beta1.Deployment, error) {
|
||||
return deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Get(name, metav1.GetOptions{})
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) updateDeployment(deployment *v1beta1.Deployment, ns string) error {
|
||||
_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Update(deployment)
|
||||
return err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) deleteDeployment(ns string, name string) error {
|
||||
// DeletePropagationBackground deletes the object immediately and dependent are deleted later
|
||||
// DeletePropagationForeground not advisable; it markes for deleteion and API can still serve those objects
|
||||
deletePropagation := metav1.DeletePropagationBackground
|
||||
err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(ns).Delete(name, &metav1.DeleteOptions{
|
||||
PropagationPolicy: &deletePropagation,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getDeploymentSpec(fn *fv1.Function, env *fv1.Environment,
|
||||
deployName string, deployLabels map[string]string) (*v1beta1.Deployment, error) {
|
||||
|
||||
replicas := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
|
||||
|
||||
gracePeriodSeconds := int64(6 * 60)
|
||||
if env.Spec.TerminationGracePeriod > 0 {
|
||||
gracePeriodSeconds = env.Spec.TerminationGracePeriod
|
||||
}
|
||||
|
||||
podAnnotations := env.Metadata.Annotations
|
||||
if podAnnotations == nil {
|
||||
podAnnotations = make(map[string]string)
|
||||
}
|
||||
if deploy.useIstio && env.Spec.AllowAccessToExternalNetwork {
|
||||
podAnnotations["sidecar.istio.io/inject"] = "false"
|
||||
}
|
||||
resources := deploy.getResources(env, fn)
|
||||
|
||||
deployment := &v1beta1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: deployName,
|
||||
Labels: deployLabels,
|
||||
},
|
||||
Spec: v1beta1.DeploymentSpec{
|
||||
Replicas: &replicas,
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: deployLabels,
|
||||
},
|
||||
Template: apiv1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: deployLabels,
|
||||
Annotations: podAnnotations,
|
||||
},
|
||||
Spec: apiv1.PodSpec{
|
||||
Containers: []apiv1.Container{
|
||||
util.MergeContainerSpecs(&apiv1.Container{
|
||||
Name: fn.Metadata.Name,
|
||||
Image: env.Spec.Runtime.Image,
|
||||
ImagePullPolicy: deploy.runtimeImagePullPolicy,
|
||||
TerminationMessagePath: "/dev/termination-log",
|
||||
Lifecycle: &apiv1.Lifecycle{
|
||||
PreStop: &apiv1.Handler{
|
||||
Exec: &apiv1.ExecAction{
|
||||
Command: []string{
|
||||
"/bin/sleep",
|
||||
fmt.Sprintf("%v", gracePeriodSeconds),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Resources: resources,
|
||||
}, env.Spec.Runtime.Container),
|
||||
},
|
||||
ServiceAccountName: "fission-fetcher",
|
||||
TerminationGracePeriodSeconds: &gracePeriodSeconds,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Order of merging is important here - first fetcher, then containers and lastly pod spec
|
||||
err := deploy.fetcherConfig.AddSpecializingFetcherToPodSpec(
|
||||
&deployment.Spec.Template.Spec,
|
||||
fn.Metadata.Name,
|
||||
fn,
|
||||
env,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if env.Spec.Runtime.PodSpec != nil {
|
||||
err := util.MergePodSpec(&deployment.Spec.Template.Spec, env.Spec.Runtime.PodSpec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return deployment, nil
|
||||
}
|
||||
|
||||
// getResources overrides only the resources which are overridden at function level otherwise
|
||||
// default to resources specified at environment level
|
||||
func (deploy *NewDeploy) getResources(env *fv1.Environment, fn *fv1.Function) apiv1.ResourceRequirements {
|
||||
resources := env.Spec.Resources
|
||||
if resources.Requests == nil {
|
||||
resources.Requests = make(map[apiv1.ResourceName]resource.Quantity)
|
||||
}
|
||||
if resources.Limits == nil {
|
||||
resources.Limits = make(map[apiv1.ResourceName]resource.Quantity)
|
||||
}
|
||||
// Only override the once specified at function, rest default to values from env.
|
||||
val, ok := fn.Spec.Resources.Requests[apiv1.ResourceCPU]
|
||||
if ok && !val.IsZero() {
|
||||
resources.Requests[apiv1.ResourceCPU] = fn.Spec.Resources.Requests[apiv1.ResourceCPU]
|
||||
}
|
||||
|
||||
val, ok = fn.Spec.Resources.Requests[apiv1.ResourceMemory]
|
||||
if ok && !val.IsZero() {
|
||||
resources.Requests[apiv1.ResourceMemory] = fn.Spec.Resources.Requests[apiv1.ResourceMemory]
|
||||
}
|
||||
|
||||
val, ok = fn.Spec.Resources.Limits[apiv1.ResourceCPU]
|
||||
if ok && !val.IsZero() {
|
||||
resources.Limits[apiv1.ResourceCPU] = fn.Spec.Resources.Limits[apiv1.ResourceCPU]
|
||||
}
|
||||
|
||||
val, ok = fn.Spec.Resources.Limits[apiv1.ResourceMemory]
|
||||
if ok && !val.IsZero() {
|
||||
resources.Limits[apiv1.ResourceMemory] = fn.Spec.Resources.Limits[apiv1.ResourceMemory]
|
||||
}
|
||||
|
||||
return resources
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) createOrGetHpa(hpaName string, execStrategy *fv1.ExecutionStrategy, depl *v1beta1.Deployment) (*asv1.HorizontalPodAutoscaler, error) {
|
||||
|
||||
minRepl := int32(execStrategy.MinScale)
|
||||
if minRepl == 0 {
|
||||
minRepl = 1
|
||||
}
|
||||
maxRepl := int32(execStrategy.MaxScale)
|
||||
if maxRepl == 0 {
|
||||
maxRepl = minRepl
|
||||
}
|
||||
targetCPU := int32(execStrategy.TargetCPUPercent)
|
||||
|
||||
existingHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Get(hpaName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
return existingHpa, err
|
||||
}
|
||||
|
||||
if depl == nil {
|
||||
return nil, errors.New("failed to create HPA, found empty deployment")
|
||||
}
|
||||
|
||||
if err != nil && k8s_err.IsNotFound(err) {
|
||||
hpa := asv1.HorizontalPodAutoscaler{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: hpaName,
|
||||
Labels: depl.Labels,
|
||||
},
|
||||
Spec: asv1.HorizontalPodAutoscalerSpec{
|
||||
ScaleTargetRef: asv1.CrossVersionObjectReference{
|
||||
Kind: DeploymentKind,
|
||||
Name: depl.ObjectMeta.Name,
|
||||
APIVersion: DeploymentVersion,
|
||||
},
|
||||
MinReplicas: &minRepl,
|
||||
MaxReplicas: maxRepl,
|
||||
TargetCPUUtilizationPercentage: &targetCPU,
|
||||
},
|
||||
}
|
||||
|
||||
cHpa, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(depl.ObjectMeta.Namespace).Create(&hpa)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cHpa, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getHpa(ns, name string) (*asv1.HorizontalPodAutoscaler, error) {
|
||||
return deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(ns).Get(name, metav1.GetOptions{})
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) updateHpa(hpa *asv1.HorizontalPodAutoscaler) error {
|
||||
_, err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Update(hpa)
|
||||
return err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) deleteHpa(ns string, name string) error {
|
||||
err := deploy.kubernetesClient.AutoscalingV1().HorizontalPodAutoscalers(ns).Delete(name, &metav1.DeleteOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) createOrGetSvc(deployLabels map[string]string, svcName string, svcNamespace string) (*apiv1.Service, error) {
|
||||
|
||||
existingSvc, err := deploy.kubernetesClient.CoreV1().Services(svcNamespace).Get(svcName, metav1.GetOptions{})
|
||||
if err == nil {
|
||||
return existingSvc, err
|
||||
}
|
||||
|
||||
if err != nil && k8s_err.IsNotFound(err) {
|
||||
service := &apiv1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: svcName,
|
||||
Labels: deployLabels,
|
||||
},
|
||||
Spec: apiv1.ServiceSpec{
|
||||
Ports: []apiv1.ServicePort{
|
||||
{
|
||||
Name: "runtime-env-port",
|
||||
Port: int32(80),
|
||||
TargetPort: intstr.FromInt(8888),
|
||||
},
|
||||
{
|
||||
Name: "fetcher-port",
|
||||
Port: int32(8000),
|
||||
TargetPort: intstr.FromInt(8000),
|
||||
},
|
||||
},
|
||||
Selector: deployLabels,
|
||||
Type: apiv1.ServiceTypeClusterIP,
|
||||
},
|
||||
}
|
||||
|
||||
svc, err := deploy.kubernetesClient.CoreV1().Services(svcNamespace).Create(service)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return svc, nil
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) deleteSvc(ns string, name string) error {
|
||||
err := deploy.kubernetesClient.CoreV1().Services(ns).Delete(name, &metav1.DeleteOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) waitForDeploy(depl *v1beta1.Deployment, replicas int32) (*v1beta1.Deployment, error) {
|
||||
for i := 0; i < 120; i++ {
|
||||
latestDepl, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(depl.ObjectMeta.Namespace).Get(depl.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
//TODO check for imagePullerror
|
||||
// use AvailableReplicas here is better than ReadyReplicas
|
||||
// since the pods may not be able to serve network traffic yet.
|
||||
if latestDepl.Status.AvailableReplicas >= replicas {
|
||||
return latestDepl, err
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
return nil, errors.New("failed to create deployment within timeout window")
|
||||
}
|
||||
|
||||
// cleanupNewdeploy cleans all kubernetes objects related to function
|
||||
func (deploy *NewDeploy) cleanupNewdeploy(ns string, name string) error {
|
||||
var multierr *multierror.Error
|
||||
|
||||
err := deploy.deleteSvc(ns, name)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error deleting service for newdeploy function",
|
||||
zap.Error(err),
|
||||
zap.String("function_name", name),
|
||||
zap.String("function_namespace", ns))
|
||||
multierror.Append(multierr, err)
|
||||
}
|
||||
|
||||
err = deploy.deleteHpa(ns, name)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error deleting service for newdeploy function",
|
||||
zap.Error(err),
|
||||
zap.String("function_name", name),
|
||||
zap.String("function_namespace", ns))
|
||||
multierror.Append(multierr, err)
|
||||
}
|
||||
|
||||
err = deploy.deleteDeployment(ns, name)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error deleting deployment for newdeploy function",
|
||||
zap.Error(err),
|
||||
zap.String("function_name", name),
|
||||
zap.String("function_namespace", ns))
|
||||
multierror.Append(multierr, err)
|
||||
}
|
||||
return multierr.ErrorOrNil()
|
||||
}
|
||||
@@ -0,0 +1,722 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package newdeploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dchest/uniuri"
|
||||
"github.com/fission/fission/pkg/throttler"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/pkg/errors"
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
"k8s.io/api/extensions/v1beta1"
|
||||
k8sErrs "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
k8sTypes "k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
"github.com/fission/fission/pkg/executor/fscache"
|
||||
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
|
||||
"github.com/fission/fission/pkg/types"
|
||||
)
|
||||
|
||||
type (
|
||||
NewDeploy struct {
|
||||
logger *zap.Logger
|
||||
|
||||
kubernetesClient *kubernetes.Clientset
|
||||
fissionClient *crd.FissionClient
|
||||
crdClient *rest.RESTClient
|
||||
instanceID string
|
||||
fetcherConfig *fetcherConfig.Config
|
||||
|
||||
runtimeImagePullPolicy apiv1.PullPolicy
|
||||
namespace string
|
||||
useIstio bool
|
||||
collectorEndpoint string
|
||||
|
||||
fsCache *fscache.FunctionServiceCache // cache funcSvc's by function, address and pod name
|
||||
|
||||
throttler *throttler.Throttler
|
||||
funcStore k8sCache.Store
|
||||
funcController k8sCache.Controller
|
||||
|
||||
envStore k8sCache.Store
|
||||
envController k8sCache.Controller
|
||||
|
||||
idlePodReapTime time.Duration
|
||||
}
|
||||
)
|
||||
|
||||
func MakeNewDeploy(
|
||||
logger *zap.Logger,
|
||||
fissionClient *crd.FissionClient,
|
||||
kubernetesClient *kubernetes.Clientset,
|
||||
crdClient *rest.RESTClient,
|
||||
namespace string,
|
||||
fetcherConfig *fetcherConfig.Config,
|
||||
instanceID string,
|
||||
) *NewDeploy {
|
||||
|
||||
logger.Info("creating NewDeploy ExecutorType")
|
||||
|
||||
enableIstio := false
|
||||
if len(os.Getenv("ENABLE_ISTIO")) > 0 {
|
||||
istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
|
||||
if err != nil {
|
||||
logger.Error("failed to parse 'ENABLE_ISTIO', set to false", zap.Error(err))
|
||||
}
|
||||
enableIstio = istio
|
||||
}
|
||||
|
||||
nd := &NewDeploy{
|
||||
logger: logger.Named("new_deploy"),
|
||||
|
||||
fissionClient: fissionClient,
|
||||
kubernetesClient: kubernetesClient,
|
||||
crdClient: crdClient,
|
||||
instanceID: instanceID,
|
||||
|
||||
namespace: namespace,
|
||||
fsCache: fscache.MakeFunctionServiceCache(logger),
|
||||
throttler: throttler.MakeThrottler(1 * time.Minute),
|
||||
|
||||
fetcherConfig: fetcherConfig,
|
||||
runtimeImagePullPolicy: utils.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY")),
|
||||
useIstio: enableIstio,
|
||||
|
||||
idlePodReapTime: 2 * time.Minute,
|
||||
}
|
||||
|
||||
if nd.crdClient != nil {
|
||||
fnStore, fnController := nd.initFuncController()
|
||||
nd.funcStore = fnStore
|
||||
nd.funcController = fnController
|
||||
|
||||
envStore, envController := nd.initEnvController()
|
||||
nd.envStore = envStore
|
||||
nd.envController = envController
|
||||
}
|
||||
|
||||
return nd
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) Run(ctx context.Context) {
|
||||
//go deploy.service()
|
||||
go deploy.funcController.Run(ctx.Done())
|
||||
go deploy.envController.Run(ctx.Done())
|
||||
go deploy.idleObjectReaper()
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) initFuncController() (k8sCache.Store, k8sCache.Controller) {
|
||||
resyncPeriod := 30 * time.Second
|
||||
listWatch := k8sCache.NewListWatchFromClient(deploy.crdClient, "functions", metav1.NamespaceAll, fields.Everything())
|
||||
store, controller := k8sCache.NewInformer(listWatch, &fv1.Function{}, resyncPeriod, k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
fn := obj.(*fv1.Function)
|
||||
_, err := deploy.createFunction(fn, true)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error eager creating function",
|
||||
zap.Error(err),
|
||||
zap.Any("function", fn))
|
||||
}
|
||||
},
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
fn := obj.(*fv1.Function)
|
||||
err := deploy.deleteFunction(fn)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error deleting function",
|
||||
zap.Error(err),
|
||||
zap.Any("function", fn))
|
||||
}
|
||||
},
|
||||
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
||||
oldFn := oldObj.(*fv1.Function)
|
||||
newFn := newObj.(*fv1.Function)
|
||||
err := deploy.updateFunction(oldFn, newFn)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error updating function",
|
||||
zap.Error(err),
|
||||
zap.Any("old_function", oldFn),
|
||||
zap.Any("new_function", newFn))
|
||||
}
|
||||
},
|
||||
})
|
||||
return store, controller
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) initEnvController() (k8sCache.Store, k8sCache.Controller) {
|
||||
resyncPeriod := 30 * time.Second
|
||||
listWatch := k8sCache.NewListWatchFromClient(deploy.crdClient, "environments", metav1.NamespaceAll, fields.Everything())
|
||||
store, controller := k8sCache.NewInformer(listWatch, &fv1.Environment{}, resyncPeriod, k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {},
|
||||
DeleteFunc: func(obj interface{}) {},
|
||||
UpdateFunc: func(oldObj interface{}, newObj interface{}) {
|
||||
newEnv := newObj.(*fv1.Environment)
|
||||
oldEnv := oldObj.(*fv1.Environment)
|
||||
// Currently only an image update in environment calls for function's deployment recreation. In future there might be more attributes which would want to do it
|
||||
if oldEnv.Spec.Runtime.Image != newEnv.Spec.Runtime.Image {
|
||||
deploy.logger.Info("Updating all function of the environment that changed, old env:", zap.Any("environment", oldEnv))
|
||||
funcs := deploy.getEnvFunctions(&newEnv.Metadata)
|
||||
for _, f := range funcs {
|
||||
function, err := deploy.fissionClient.Functions(f.Metadata.Namespace).Get(f.Metadata.Name)
|
||||
if err != nil {
|
||||
deploy.logger.Error("Error getting function", zap.Error(err), zap.Any("function", function))
|
||||
}
|
||||
err = deploy.updateFuncDeployment(function, newEnv)
|
||||
if err != nil {
|
||||
deploy.logger.Error("Error updating function", zap.Error(err), zap.Any("function", function))
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
return store, controller
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getEnvFunctions(m *metav1.ObjectMeta) []fv1.Function {
|
||||
funcList, err := deploy.fissionClient.Functions(m.Namespace).List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Error("Error getting functions for env", zap.Error(err), zap.Any("environment", m))
|
||||
}
|
||||
relatedFunctions := make([]fv1.Function, 0)
|
||||
for _, f := range funcList.Items {
|
||||
if (f.Spec.Environment.Name == m.Name) && (f.Spec.Environment.Namespace == m.Namespace) {
|
||||
relatedFunctions = append(relatedFunctions, f)
|
||||
}
|
||||
}
|
||||
return relatedFunctions
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
|
||||
fn, err := deploy.fissionClient.Functions(metadata.Namespace).Get(metadata.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return deploy.createFunction(fn, false)
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) createFunction(fn *fv1.Function, firstcreate bool) (*fscache.FuncSvc, error) {
|
||||
if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
fsvcObj, err := deploy.throttler.RunOnce(string(fn.Metadata.UID), func(ableToCreate bool) (interface{}, error) {
|
||||
if ableToCreate {
|
||||
return deploy.fnCreate(fn, firstcreate)
|
||||
}
|
||||
return deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
|
||||
})
|
||||
|
||||
fsvc, ok := fsvcObj.(*fscache.FuncSvc)
|
||||
if !ok {
|
||||
deploy.logger.Panic("receive unknown object while creating function - expected pointer of function service object")
|
||||
}
|
||||
|
||||
return fsvc, err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) deleteFunction(fn *fv1.Function) error {
|
||||
if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
|
||||
return nil
|
||||
}
|
||||
err := deploy.fnDelete(fn)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "error deleting kubernetes objects of function %v", fn.Metadata)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) fnCreate(fn *fv1.Function, firstcreate bool) (*fscache.FuncSvc, error) {
|
||||
env, err := deploy.fissionClient.
|
||||
Environments(fn.Spec.Environment.Namespace).
|
||||
Get(fn.Spec.Environment.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objName := deploy.getObjName(fn)
|
||||
if !firstcreate {
|
||||
// retrieve back the previous obj name for later use.
|
||||
fsvc, err := deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
|
||||
if err == nil {
|
||||
objName = fsvc.Name
|
||||
}
|
||||
}
|
||||
deployLabels := deploy.getDeployLabels(fn, env)
|
||||
|
||||
// to support backward compatibility, if the function was created in default ns, we fall back to creating the
|
||||
// deployment of the function in fission-function ns
|
||||
ns := deploy.namespace
|
||||
if fn.Metadata.Namespace != metav1.NamespaceDefault {
|
||||
ns = fn.Metadata.Namespace
|
||||
}
|
||||
|
||||
// Envoy(istio-proxy) returns 404 directly before istio pilot
|
||||
// propagates latest Envoy-specific configuration.
|
||||
// Since newdeploy waits for pods of deployment to be ready,
|
||||
// change the order of kubeObject creation (create service first,
|
||||
// then deployment) to take advantage of waiting time.
|
||||
svc, err := deploy.createOrGetSvc(deployLabels, objName, ns)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating service", zap.Error(err), zap.String("service", objName))
|
||||
go deploy.cleanupNewdeploy(ns, objName)
|
||||
return nil, errors.Wrapf(err, "error creating service %v", objName)
|
||||
}
|
||||
svcAddress := fmt.Sprintf("%v.%v", svc.Name, svc.Namespace)
|
||||
depl, err := deploy.createOrGetDeployment(fn, env, objName, deployLabels, ns, firstcreate)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating deployment", zap.Error(err), zap.String("deployment", objName))
|
||||
go deploy.cleanupNewdeploy(ns, objName)
|
||||
return nil, errors.Wrapf(err, "error creating deployment %v", objName)
|
||||
}
|
||||
|
||||
hpa, err := deploy.createOrGetHpa(objName, &fn.Spec.InvokeStrategy.ExecutionStrategy, depl)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error creating HPA", zap.Error(err), zap.String("hpa", objName))
|
||||
go deploy.cleanupNewdeploy(ns, objName)
|
||||
return nil, errors.Wrapf(err, "error creating the HPA %v", objName)
|
||||
}
|
||||
|
||||
kubeObjRefs := []apiv1.ObjectReference{
|
||||
{
|
||||
//obj.TypeMeta.Kind does not work hence this, needs investigation and a fix
|
||||
Kind: "deployment",
|
||||
Name: depl.ObjectMeta.Name,
|
||||
APIVersion: depl.TypeMeta.APIVersion,
|
||||
Namespace: depl.ObjectMeta.Namespace,
|
||||
ResourceVersion: depl.ObjectMeta.ResourceVersion,
|
||||
UID: depl.ObjectMeta.UID,
|
||||
},
|
||||
{
|
||||
Kind: "service",
|
||||
Name: svc.ObjectMeta.Name,
|
||||
APIVersion: svc.TypeMeta.APIVersion,
|
||||
Namespace: svc.ObjectMeta.Namespace,
|
||||
ResourceVersion: svc.ObjectMeta.ResourceVersion,
|
||||
UID: svc.ObjectMeta.UID,
|
||||
},
|
||||
{
|
||||
Kind: "horizontalpodautoscaler",
|
||||
Name: hpa.ObjectMeta.Name,
|
||||
APIVersion: hpa.TypeMeta.APIVersion,
|
||||
Namespace: hpa.ObjectMeta.Namespace,
|
||||
ResourceVersion: hpa.ObjectMeta.ResourceVersion,
|
||||
UID: hpa.ObjectMeta.UID,
|
||||
},
|
||||
}
|
||||
|
||||
fsvc := &fscache.FuncSvc{
|
||||
Name: objName,
|
||||
Function: &fn.Metadata,
|
||||
Environment: env,
|
||||
Address: svcAddress,
|
||||
KubernetesObjects: kubeObjRefs,
|
||||
Executor: fscache.NEWDEPLOY,
|
||||
}
|
||||
|
||||
_, err = deploy.fsCache.Add(*fsvc)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error adding function to cache", zap.Error(err), zap.Any("function", fsvc.Function))
|
||||
return fsvc, err
|
||||
}
|
||||
return fsvc, nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) updateFunction(oldFn *fv1.Function, newFn *fv1.Function) error {
|
||||
|
||||
if oldFn.Metadata.ResourceVersion == newFn.Metadata.ResourceVersion {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Ignoring updates to functions which are not of NewDeployment type
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy &&
|
||||
oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Executor type is no longer New Deployment
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy &&
|
||||
oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypeNewdeploy {
|
||||
deploy.logger.Info("function does not use new deployment executor anymore, deleting resources",
|
||||
zap.Any("function", newFn))
|
||||
// IMP - pass the oldFn, as the new/modified function is not in cache
|
||||
return deploy.deleteFunction(oldFn)
|
||||
}
|
||||
|
||||
// Executor type changed to New Deployment from something else
|
||||
if oldFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypeNewdeploy &&
|
||||
newFn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypeNewdeploy {
|
||||
deploy.logger.Info("function type changed to new deployment, creating resources",
|
||||
zap.Any("old_function", oldFn.Metadata),
|
||||
zap.Any("new_function", newFn.Metadata))
|
||||
_, err := deploy.createFunction(newFn, true)
|
||||
if err != nil {
|
||||
deploy.updateStatus(oldFn, err, "error changing the function's type to newdeploy")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
deployChanged := false
|
||||
|
||||
if oldFn.Spec.InvokeStrategy != newFn.Spec.InvokeStrategy {
|
||||
|
||||
// to support backward compatibility, if the function was created in default ns, we fall back to creating the
|
||||
// deployment of the function in fission-function ns, so cleaning up resources there
|
||||
ns := deploy.namespace
|
||||
if newFn.Metadata.Namespace != metav1.NamespaceDefault {
|
||||
ns = newFn.Metadata.Namespace
|
||||
}
|
||||
|
||||
fsvc, err := deploy.fsCache.GetByFunctionUID(newFn.Metadata.UID)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "error updating function due to unable to find function service cache: %v", oldFn)
|
||||
return err
|
||||
}
|
||||
|
||||
hpa, err := deploy.getHpa(ns, fsvc.Name)
|
||||
if err != nil {
|
||||
deploy.updateStatus(oldFn, err, "error getting HPA while updating function")
|
||||
return err
|
||||
}
|
||||
|
||||
hpaChanged := false
|
||||
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale {
|
||||
replicas := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
|
||||
hpa.Spec.MinReplicas = &replicas
|
||||
hpaChanged = true
|
||||
}
|
||||
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale != oldFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale {
|
||||
hpa.Spec.MaxReplicas = int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.MaxScale)
|
||||
hpaChanged = true
|
||||
}
|
||||
|
||||
if newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent != oldFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent {
|
||||
targetCpupercent := int32(newFn.Spec.InvokeStrategy.ExecutionStrategy.TargetCPUPercent)
|
||||
hpa.Spec.TargetCPUUtilizationPercentage = &targetCpupercent
|
||||
hpaChanged = true
|
||||
}
|
||||
|
||||
if hpaChanged {
|
||||
err := deploy.updateHpa(hpa)
|
||||
if err != nil {
|
||||
deploy.updateStatus(oldFn, err, "error updating HPA while updating function")
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if oldFn.Spec.Environment != newFn.Spec.Environment ||
|
||||
oldFn.Spec.Package.PackageRef != newFn.Spec.Package.PackageRef ||
|
||||
oldFn.Spec.Package.FunctionName != newFn.Spec.Package.FunctionName {
|
||||
deployChanged = true
|
||||
}
|
||||
|
||||
// If length of slice has changed then no need to check individual elements
|
||||
if len(oldFn.Spec.Secrets) != len(newFn.Spec.Secrets) {
|
||||
deployChanged = true
|
||||
} else {
|
||||
for i, newSecret := range newFn.Spec.Secrets {
|
||||
if newSecret != oldFn.Spec.Secrets[i] {
|
||||
deployChanged = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(oldFn.Spec.ConfigMaps) != len(newFn.Spec.ConfigMaps) {
|
||||
deployChanged = true
|
||||
} else {
|
||||
for i, newConfig := range newFn.Spec.ConfigMaps {
|
||||
if newConfig != oldFn.Spec.ConfigMaps[i] {
|
||||
deployChanged = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if deployChanged == true {
|
||||
env, err := deploy.fissionClient.Environments(newFn.Spec.Environment.Namespace).
|
||||
Get(newFn.Spec.Environment.Name)
|
||||
if err != nil {
|
||||
deploy.updateStatus(oldFn, err, "failed to get environment while updating function")
|
||||
return err
|
||||
}
|
||||
return deploy.updateFuncDeployment(newFn, env)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) updateFuncDeployment(fn *fv1.Function, env *fv1.Environment) error {
|
||||
|
||||
fsvc, err := deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
|
||||
if err != nil {
|
||||
err = errors.Wrapf(err, "error updating function due to unable to find function service cache: %v", fn)
|
||||
return err
|
||||
}
|
||||
fnObjName := fsvc.Name
|
||||
|
||||
deployLabels := deploy.getDeployLabels(fn, env)
|
||||
deploy.logger.Info("updating deployment due to function/environment update", zap.String("deployment", fnObjName), zap.Any("function", fn.Metadata.Name))
|
||||
|
||||
newDeployment, err := deploy.getDeploymentSpec(fn, env, fnObjName, deployLabels)
|
||||
if err != nil {
|
||||
deploy.updateStatus(fn, err, "failed to get new deployment spec while updating function")
|
||||
return err
|
||||
}
|
||||
|
||||
// to support backward compatibility, if the function was created in default ns, we fall back to creating the
|
||||
// deployment of the function in fission-function ns
|
||||
ns := deploy.namespace
|
||||
if fn.Metadata.Namespace != metav1.NamespaceDefault {
|
||||
ns = fn.Metadata.Namespace
|
||||
}
|
||||
|
||||
err = deploy.updateDeployment(newDeployment, ns)
|
||||
if err != nil {
|
||||
deploy.updateStatus(fn, err, "failed to update deployment while updating function")
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) fnDelete(fn *fv1.Function) error {
|
||||
var multierr *multierror.Error
|
||||
|
||||
// GetByFunction uses resource version as part of cache key, however,
|
||||
// the resource version in function metadata will be changed when a function
|
||||
// is deleted and cause newdeploy backend fails to delete the entry.
|
||||
// Use GetByFunctionUID instead of GetByFunction here to find correct
|
||||
// fsvc entry.
|
||||
fsvc, err := deploy.fsCache.GetByFunctionUID(fn.Metadata.UID)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, fmt.Sprintf("fsvc not found in cache: %v", fn.Metadata))
|
||||
return err
|
||||
}
|
||||
|
||||
objName := fsvc.Name
|
||||
|
||||
_, err = deploy.fsCache.DeleteOld(fsvc, time.Second*0)
|
||||
if err != nil {
|
||||
multierr = multierror.Append(multierr,
|
||||
errors.Wrap(err, fmt.Sprintf("error deleting the function from cache")))
|
||||
}
|
||||
|
||||
// to support backward compatibility, if the function was created in default ns, we fall back to creating the
|
||||
// deployment of the function in fission-function ns, so cleaning up resources there
|
||||
ns := deploy.namespace
|
||||
if fn.Metadata.Namespace != metav1.NamespaceDefault {
|
||||
ns = fn.Metadata.Namespace
|
||||
}
|
||||
|
||||
err = deploy.cleanupNewdeploy(ns, objName)
|
||||
multierr = multierror.Append(multierr, err)
|
||||
|
||||
return multierr.ErrorOrNil()
|
||||
}
|
||||
|
||||
// getObjName returns a unique name for kubernetes objects of function
|
||||
func (deploy *NewDeploy) getObjName(fn *fv1.Function) string {
|
||||
return strings.ToLower(fmt.Sprintf("newdeploy-%v-%v-%v", fn.Metadata.Name, fn.Metadata.Namespace, uniuri.NewLen(8)))
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) getDeployLabels(fn *fv1.Function, env *fv1.Environment) map[string]string {
|
||||
return map[string]string{
|
||||
types.EXECUTOR_INSTANCEID_LABEL: deploy.instanceID,
|
||||
types.EXECUTOR_TYPE: fv1.ExecutorTypeNewdeploy,
|
||||
types.ENVIRONMENT_NAME: env.Metadata.Name,
|
||||
types.ENVIRONMENT_NAMESPACE: env.Metadata.Namespace,
|
||||
types.ENVIRONMENT_UID: string(env.Metadata.UID),
|
||||
types.FUNCTION_NAME: fn.Metadata.Name,
|
||||
types.FUNCTION_NAMESPACE: fn.Metadata.Namespace,
|
||||
types.FUNCTION_UID: string(fn.Metadata.UID),
|
||||
}
|
||||
}
|
||||
|
||||
// updateKubeObjRefRV update the resource version of kubeObjectRef with
|
||||
// given kind and return error if failed to find the reference.
|
||||
func (deploy *NewDeploy) updateKubeObjRefRV(fsvc *fscache.FuncSvc, objKind string, rv string) error {
|
||||
kubeObjs := fsvc.KubernetesObjects
|
||||
for i, obj := range kubeObjs {
|
||||
if obj.Kind == objKind {
|
||||
kubeObjs[i].ResourceVersion = rv
|
||||
return nil
|
||||
}
|
||||
}
|
||||
fsvc.KubernetesObjects = kubeObjs
|
||||
return fmt.Errorf("error finding kubernetes object reference with kind: %v", objKind)
|
||||
}
|
||||
|
||||
// updateStatus is a function which updates status of update.
|
||||
// Current implementation only logs messages, in future it will update function status
|
||||
func (deploy *NewDeploy) updateStatus(fn *fv1.Function, err error, message string) {
|
||||
deploy.logger.Info("function status update", zap.Error(err), zap.Any("function", fn), zap.String("message", message))
|
||||
}
|
||||
|
||||
// IsValid does a get on the service address to ensure it's a valid service, then
|
||||
// scale deployment to 1 replica if there are no available replicas for function.
|
||||
// Return true if no error occurs, return false otherwise.
|
||||
func (deploy *NewDeploy) IsValid(fsvc *fscache.FuncSvc) bool {
|
||||
service := strings.Split(fsvc.Address, ".")
|
||||
if len(service) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
_, err := deploy.kubernetesClient.CoreV1().Services(service[1]).Get(service[0], metav1.GetOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Error("error validating function service address", zap.String("function", fsvc.Function.Name), zap.Error(err))
|
||||
return false
|
||||
}
|
||||
|
||||
deployObj := getDeploymentObj(fsvc.KubernetesObjects)
|
||||
if deployObj == nil {
|
||||
deploy.logger.Error("deployment obj for function does not exist", zap.String("function", fsvc.Function.Name))
|
||||
return false
|
||||
}
|
||||
|
||||
currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
|
||||
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
return false
|
||||
}
|
||||
|
||||
// return directly when available replicas > 0
|
||||
if currentDeploy.Status.AvailableReplicas > 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// idleObjectReaper reaps objects after certain idle time
|
||||
func (deploy *NewDeploy) idleObjectReaper() {
|
||||
|
||||
pollSleep := time.Duration(deploy.idlePodReapTime)
|
||||
for {
|
||||
time.Sleep(pollSleep)
|
||||
|
||||
envs, err := deploy.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Fatal("failed to get environment list", zap.Error(err))
|
||||
}
|
||||
|
||||
envList := make(map[k8sTypes.UID]struct{})
|
||||
for _, env := range envs.Items {
|
||||
envList[env.Metadata.UID] = struct{}{}
|
||||
}
|
||||
|
||||
funcSvcs, err := deploy.fsCache.ListOld(deploy.idlePodReapTime)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error reaping idle pods", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
for _, fsvc := range funcSvcs {
|
||||
if fsvc.Executor != fscache.NEWDEPLOY {
|
||||
continue
|
||||
}
|
||||
|
||||
// For function with the environment that no longer exists, executor
|
||||
// scales down the deployment as usual and prints log to notify user.
|
||||
if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
|
||||
deploy.logger.Error("function environment no longer exists",
|
||||
zap.String("environment", fsvc.Environment.Metadata.Name),
|
||||
zap.String("function", fsvc.Name))
|
||||
}
|
||||
|
||||
fn, err := deploy.fissionClient.Functions(fsvc.Function.Namespace).Get(fsvc.Function.Name)
|
||||
if err != nil {
|
||||
// Newdeploy manager handles the function delete event and clean cache/kubeobjs itself,
|
||||
// so we ignore the not found error for functions with newdeploy executor type here.
|
||||
if k8sErrs.IsNotFound(err) && fsvc.Executor == fscache.NEWDEPLOY {
|
||||
continue
|
||||
}
|
||||
deploy.logger.Error("error getting function", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
continue
|
||||
}
|
||||
|
||||
deployObj := getDeploymentObj(fsvc.KubernetesObjects)
|
||||
if deployObj == nil {
|
||||
deploy.logger.Error("error finding function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
continue
|
||||
}
|
||||
|
||||
currentDeploy, err := deploy.kubernetesClient.ExtensionsV1beta1().
|
||||
Deployments(deployObj.Namespace).Get(deployObj.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
deploy.logger.Error("error validating function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
continue
|
||||
}
|
||||
|
||||
minScale := int32(fn.Spec.InvokeStrategy.ExecutionStrategy.MinScale)
|
||||
|
||||
// do nothing if the current replicas is already lower than minScale
|
||||
if *currentDeploy.Spec.Replicas <= minScale {
|
||||
continue
|
||||
}
|
||||
|
||||
err = deploy.scaleDeployment(deployObj.Namespace, deployObj.Name, minScale)
|
||||
if err != nil {
|
||||
deploy.logger.Error("error scaling down function deployment", zap.Error(err), zap.String("function", fsvc.Function.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getDeploymentObj(kubeobjs []apiv1.ObjectReference) *apiv1.ObjectReference {
|
||||
for _, kubeobj := range kubeobjs {
|
||||
switch strings.ToLower(kubeobj.Kind) {
|
||||
case "deployment":
|
||||
return &kubeobj
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (deploy *NewDeploy) scaleDeployment(deplNS string, deplName string, replicas int32) error {
|
||||
deploy.logger.Info("scaling deployment",
|
||||
zap.String("deployment", deplName),
|
||||
zap.String("namespace", deplNS),
|
||||
zap.Int32("replicas", replicas))
|
||||
_, err := deploy.kubernetesClient.ExtensionsV1beta1().Deployments(deplNS).UpdateScale(deplName, &v1beta1.Scale{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: deplName,
|
||||
Namespace: deplNS,
|
||||
},
|
||||
Spec: v1beta1.ScaleSpec{
|
||||
Replicas: replicas,
|
||||
},
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
/*
|
||||
Copyright 2018 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package poolmgr
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/fission/fission/pkg/types"
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
kerrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
)
|
||||
|
||||
func getIstioServiceLabels(fnName string) map[string]string {
|
||||
return map[string]string{
|
||||
"functionName": fnName,
|
||||
}
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) makeFuncController(fissionClient *crd.FissionClient,
|
||||
kubernetesClient *kubernetes.Clientset, fissionfnNamespace string, istioEnabled bool) (k8sCache.Store, k8sCache.Controller) {
|
||||
|
||||
resyncPeriod := 30 * time.Second
|
||||
lw := k8sCache.NewListWatchFromClient(fissionClient.GetCrdClient(), "functions", metav1.NamespaceAll, fields.Everything())
|
||||
|
||||
funcStore, controller := k8sCache.NewInformer(lw, &fv1.Function{}, resyncPeriod,
|
||||
k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
fn := obj.(*fv1.Function)
|
||||
|
||||
// Since istio only allows accessing pod through k8s service,
|
||||
// for the functions with executor type "poolmgr" we need to
|
||||
// create a service for sending requests to pod in pool.
|
||||
// Functions with executor type "Newdeploy" is specialized at
|
||||
// pod starts. In this case, just ignore such functions.
|
||||
fnExecutorType := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
|
||||
|
||||
// In some cases, user may not enter the executorType explicitly, for example in his spec.yaml.
|
||||
// we assume it to be of type poolmgr
|
||||
if fnExecutorType != "" && fnExecutorType != fv1.ExecutorTypePoolmgr {
|
||||
return
|
||||
}
|
||||
|
||||
// create or update role-binding
|
||||
envNs := fissionfnNamespace
|
||||
if fn.Spec.Environment.Namespace != metav1.NamespaceDefault {
|
||||
envNs = fn.Spec.Environment.Namespace
|
||||
}
|
||||
|
||||
// TODO : Just bring to your attention during review :
|
||||
// setup rolebinding is tried, if it fails, we dont return. we just log an error and move on, because :
|
||||
// 1. not all functions have secrets and/or configmaps, so things will work without this rolebinding in that case.
|
||||
// 2. on the contrary, when the route is tried, the env fetcher logs will show a 403 forbidden message and same will be relayed to executor.
|
||||
err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.SecretConfigMapGetterRB, fn.Metadata.Namespace, types.SecretConfigMapGetterCR, types.ClusterRole, types.FissionFetcherSA, envNs)
|
||||
if err != nil {
|
||||
gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", types.SecretConfigMapGetterRB))
|
||||
} else {
|
||||
gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
|
||||
zap.String("service_account", types.FissionFetcherSA),
|
||||
zap.String("service_account_namepsace", envNs),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.String("function_namespace", fn.Metadata.Namespace))
|
||||
}
|
||||
|
||||
if istioEnabled {
|
||||
// create a same name service for function
|
||||
// since istio only allows the traffic to service
|
||||
sel := map[string]string{
|
||||
"functionName": fn.Metadata.Name,
|
||||
"functionUid": string(fn.Metadata.UID),
|
||||
}
|
||||
|
||||
svcName := utils.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
|
||||
|
||||
// service for accepting user traffic
|
||||
svc := apiv1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: envNs,
|
||||
Name: svcName,
|
||||
Labels: getIstioServiceLabels(fn.Metadata.Name),
|
||||
},
|
||||
Spec: apiv1.ServiceSpec{
|
||||
Type: apiv1.ServiceTypeClusterIP,
|
||||
Ports: []apiv1.ServicePort{
|
||||
// Service port name should begin with a recognized prefix, or the traffic will be
|
||||
// treated as TCP traffic. (https://istio.io/docs/setup/kubernetes/sidecar-injection.html)
|
||||
// Originally the ports' name are similar to "http-fetch" and "http-specialize".
|
||||
// But for istio 0.5.1, istio-proxy return unexpected 431 error with such naming.
|
||||
// https://github.com/istio/istio/issues/928
|
||||
// Workaround: remove prefix
|
||||
// TODO: prepend prefix once the bug fixed
|
||||
{
|
||||
Name: "fetch",
|
||||
Protocol: apiv1.ProtocolTCP,
|
||||
Port: 8000,
|
||||
TargetPort: intstr.FromInt(8000),
|
||||
},
|
||||
{
|
||||
Name: "specialize",
|
||||
Protocol: apiv1.ProtocolTCP,
|
||||
Port: 8888,
|
||||
TargetPort: intstr.FromInt(8888),
|
||||
},
|
||||
},
|
||||
Selector: sel,
|
||||
},
|
||||
}
|
||||
|
||||
// create function istio service if it does not exist
|
||||
_, err = kubernetesClient.CoreV1().Services(envNs).Create(&svc)
|
||||
if err != nil && !kerrors.IsAlreadyExists(err) {
|
||||
gpm.logger.Error("error creating istio service for function",
|
||||
zap.Error(err),
|
||||
zap.String("service_name", svcName),
|
||||
zap.String("function_name", fn.Metadata.Name),
|
||||
zap.Any("selectors", sel))
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
DeleteFunc: func(obj interface{}) {
|
||||
fn := obj.(*fv1.Function)
|
||||
|
||||
fnExecutorType := fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType
|
||||
if fnExecutorType != "" && fnExecutorType != fv1.ExecutorTypePoolmgr {
|
||||
return
|
||||
}
|
||||
|
||||
envNs := fissionfnNamespace
|
||||
if fn.Spec.Environment.Namespace != metav1.NamespaceDefault {
|
||||
envNs = fn.Spec.Environment.Namespace
|
||||
}
|
||||
|
||||
if istioEnabled {
|
||||
svcName := utils.GetFunctionIstioServiceName(fn.Metadata.Name, fn.Metadata.Namespace)
|
||||
// delete function istio service
|
||||
err := kubernetesClient.CoreV1().Services(envNs).Delete(svcName, nil)
|
||||
if err != nil && !kerrors.IsNotFound(err) {
|
||||
gpm.logger.Error("error deleting istio service for function",
|
||||
zap.Error(err),
|
||||
zap.String("service_name", svcName),
|
||||
zap.String("function_name", fn.Metadata.Name))
|
||||
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
UpdateFunc: func(oldObj, newObj interface{}) {
|
||||
oldFunc := oldObj.(*fv1.Function)
|
||||
newFunc := newObj.(*fv1.Function)
|
||||
|
||||
if oldFunc.Metadata.ResourceVersion == newFunc.Metadata.ResourceVersion {
|
||||
return
|
||||
}
|
||||
|
||||
envChanged := (oldFunc.Spec.Environment.Namespace != newFunc.Spec.Environment.Namespace)
|
||||
|
||||
executorTypeChangedToPM := (oldFunc.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType != fv1.ExecutorTypePoolmgr &&
|
||||
newFunc.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == fv1.ExecutorTypePoolmgr)
|
||||
|
||||
// if a func's env reference gets updated and the newly referenced env is in a different ns,
|
||||
// we need to create a rolebinding in func's ns so that the fetcher-sa in env ns has access
|
||||
// to fetch secrets and config maps from the func's ns.
|
||||
// similarly if executorType changed to Pool Manager, we now need a rolebinding in the func ns for fetcher sa
|
||||
// present in env ns because for newdeploy, the fetcher sa is in function namespace
|
||||
if envChanged || executorTypeChangedToPM {
|
||||
envNs := fissionfnNamespace
|
||||
if newFunc.Spec.Environment.Namespace != metav1.NamespaceDefault {
|
||||
envNs = newFunc.Spec.Environment.Namespace
|
||||
}
|
||||
|
||||
err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.SecretConfigMapGetterRB,
|
||||
newFunc.Metadata.Namespace, types.SecretConfigMapGetterCR, types.ClusterRole,
|
||||
types.FissionFetcherSA, envNs)
|
||||
|
||||
if err != nil {
|
||||
gpm.logger.Error("error creating rolebinding", zap.Error(err), zap.String("role_binding", types.SecretConfigMapGetterRB))
|
||||
} else {
|
||||
gpm.logger.Info("successfully set up rolebinding for fetcher service account for function",
|
||||
zap.String("service_account", types.FissionFetcherSA),
|
||||
zap.String("service_account_namepsace", envNs),
|
||||
zap.String("function_name", newFunc.Metadata.Name),
|
||||
zap.String("function_namespace", newFunc.Metadata.Namespace))
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
return funcStore, controller
|
||||
}
|
||||
@@ -0,0 +1,626 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package poolmgr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dchest/uniuri"
|
||||
"github.com/fission/fission/pkg/types"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/pkg/errors"
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
"k8s.io/api/extensions/v1beta1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/labels"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
"github.com/fission/fission/pkg/executor/fscache"
|
||||
"github.com/fission/fission/pkg/executor/util"
|
||||
fetcherClient "github.com/fission/fission/pkg/fetcher/client"
|
||||
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
|
||||
)
|
||||
|
||||
type (
|
||||
GenericPool struct {
|
||||
logger *zap.Logger
|
||||
env *fv1.Environment
|
||||
replicas int32 // num idle pods
|
||||
deployment *v1beta1.Deployment // kubernetes deployment
|
||||
namespace string // namespace to keep our resources
|
||||
functionNamespace string // fallback namespace for fission functions
|
||||
podReadyTimeout time.Duration // timeout for generic pods to become ready
|
||||
idlePodReapTime time.Duration // pods unused for idlePodReapTime are deleted
|
||||
fsCache *fscache.FunctionServiceCache // cache funcSvc's by function, address and podname
|
||||
useSvc bool // create k8s service for specialized pods
|
||||
useIstio bool
|
||||
poolInstanceId string // small random string to uniquify pod names
|
||||
runtimeImagePullPolicy apiv1.PullPolicy // pull policy for generic pool to created env deployment
|
||||
kubernetesClient *kubernetes.Clientset
|
||||
fissionClient *crd.FissionClient
|
||||
instanceId string // poolmgr instance id
|
||||
labelsForPool map[string]string
|
||||
requestChannel chan *choosePodRequest
|
||||
fetcherConfig *fetcherConfig.Config
|
||||
}
|
||||
|
||||
// serialize the choosing of pods so that choices don't conflict
|
||||
choosePodRequest struct {
|
||||
newLabels map[string]string
|
||||
responseChannel chan *choosePodResponse
|
||||
}
|
||||
choosePodResponse struct {
|
||||
pod *apiv1.Pod
|
||||
error
|
||||
}
|
||||
)
|
||||
|
||||
func MakeGenericPool(
|
||||
logger *zap.Logger,
|
||||
fissionClient *crd.FissionClient,
|
||||
kubernetesClient *kubernetes.Clientset,
|
||||
env *fv1.Environment,
|
||||
initialReplicas int32,
|
||||
namespace string,
|
||||
functionNamespace string,
|
||||
fsCache *fscache.FunctionServiceCache,
|
||||
fetcherConfig *fetcherConfig.Config,
|
||||
instanceId string,
|
||||
enableIstio bool) (*GenericPool, error) {
|
||||
|
||||
gpLogger := logger.Named("generic_pool")
|
||||
|
||||
gpLogger.Info("creating pool", zap.Any("environment", env.Metadata))
|
||||
|
||||
// TODO: in general we need to provide the user a way to configure pools. Initial
|
||||
// replicas, autoscaling params, various timeouts, etc.
|
||||
gp := &GenericPool{
|
||||
logger: gpLogger,
|
||||
env: env,
|
||||
replicas: initialReplicas, // TODO make this an env param instead?
|
||||
requestChannel: make(chan *choosePodRequest),
|
||||
fissionClient: fissionClient,
|
||||
kubernetesClient: kubernetesClient,
|
||||
namespace: namespace,
|
||||
functionNamespace: functionNamespace,
|
||||
podReadyTimeout: 5 * time.Minute, // TODO make this an env param?
|
||||
idlePodReapTime: 3 * time.Minute, // TODO make this configurable
|
||||
fsCache: fsCache,
|
||||
poolInstanceId: uniuri.NewLen(8),
|
||||
fetcherConfig: fetcherConfig,
|
||||
instanceId: instanceId,
|
||||
useSvc: false, // defaults off -- svc takes a second or more to become routable, slowing cold start
|
||||
useIstio: enableIstio, // defaults off -- istio integration requires pod relabeling and it takes a second or more to become routable, slowing cold start
|
||||
}
|
||||
|
||||
gp.runtimeImagePullPolicy = utils.GetImagePullPolicy(os.Getenv("RUNTIME_IMAGE_PULL_POLICY"))
|
||||
|
||||
// create fetcher SA in this ns, if not already created
|
||||
err := fetcherConfig.SetupServiceAccount(gp.kubernetesClient, gp.namespace, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "error creating fetcher service account in namespace %q", gp.namespace)
|
||||
}
|
||||
|
||||
// Labels for generic deployment/RS/pods.
|
||||
gp.labelsForPool = gp.getDeployLabels()
|
||||
|
||||
// create the pool
|
||||
err = gp.createPool()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gpLogger.Info("deployment created", zap.Any("environment", env.Metadata))
|
||||
|
||||
go gp.choosePodService()
|
||||
|
||||
return gp, nil
|
||||
}
|
||||
|
||||
func (gp *GenericPool) getDeployLabels() map[string]string {
|
||||
return map[string]string{
|
||||
fv1.EXECUTOR_INSTANCEID_LABEL: gp.instanceId,
|
||||
types.EXECUTOR_TYPE: fv1.ExecutorTypePoolmgr,
|
||||
types.ENVIRONMENT_NAME: gp.env.Metadata.Name,
|
||||
types.ENVIRONMENT_NAMESPACE: gp.env.Metadata.Namespace,
|
||||
types.ENVIRONMENT_UID: string(gp.env.Metadata.UID),
|
||||
"managed": "true", // this allows us to easily find pods managed by the deployment
|
||||
}
|
||||
}
|
||||
|
||||
// choosePodService serializes the choosing of pods
|
||||
func (gp *GenericPool) choosePodService() {
|
||||
for {
|
||||
select {
|
||||
case req := <-gp.requestChannel:
|
||||
pod, err := gp._choosePod(req.newLabels)
|
||||
if err != nil {
|
||||
req.responseChannel <- &choosePodResponse{error: err}
|
||||
continue
|
||||
}
|
||||
req.responseChannel <- &choosePodResponse{pod: pod}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// choosePod picks a ready pod from the pool and relabels it, waiting if necessary.
|
||||
// returns the pod API object.
|
||||
func (gp *GenericPool) choosePod(newLabels map[string]string) (*apiv1.Pod, error) {
|
||||
req := &choosePodRequest{
|
||||
newLabels: newLabels,
|
||||
responseChannel: make(chan *choosePodResponse),
|
||||
}
|
||||
gp.requestChannel <- req
|
||||
resp := <-req.responseChannel
|
||||
return resp.pod, resp.error
|
||||
}
|
||||
|
||||
// _choosePod is called serially by choosePodService
|
||||
func (gp *GenericPool) _choosePod(newLabels map[string]string) (*apiv1.Pod, error) {
|
||||
startTime := time.Now()
|
||||
for {
|
||||
// Retries took too long, error out.
|
||||
if time.Since(startTime) > gp.podReadyTimeout {
|
||||
gp.logger.Error("timed out waiting for pod", zap.Any("labels", newLabels), zap.Duration("timeout", gp.podReadyTimeout))
|
||||
return nil, errors.New("timeout: waited too long to get a ready pod")
|
||||
}
|
||||
|
||||
// Get pods; filter the ones that are ready
|
||||
podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(
|
||||
metav1.ListOptions{
|
||||
LabelSelector: labels.Set(
|
||||
gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
readyPods := make([]*apiv1.Pod, 0, len(podList.Items))
|
||||
for i := range podList.Items {
|
||||
pod := podList.Items[i]
|
||||
|
||||
// Ignore not ready pod here
|
||||
if !utils.IsReadyPod(&pod) {
|
||||
continue
|
||||
}
|
||||
|
||||
// add it to the list of ready pods
|
||||
readyPods = append(readyPods, &pod)
|
||||
}
|
||||
gp.logger.Info("found ready pods",
|
||||
zap.Any("labels", newLabels),
|
||||
zap.Int("ready_count", len(readyPods)),
|
||||
zap.Int("total", len(podList.Items)))
|
||||
|
||||
// If there are no ready pods, wait and retry.
|
||||
if len(readyPods) == 0 {
|
||||
err = gp.waitForReadyPod()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Pick a ready pod. For now just choose randomly;
|
||||
// ideally we'd care about which node it's running on,
|
||||
// and make a good scheduling decision.
|
||||
chosenPod := readyPods[rand.Intn(len(readyPods))]
|
||||
|
||||
if gp.env.Spec.AllowedFunctionsPerContainer != types.AllowedFunctionsPerContainerInfinite {
|
||||
// Relabel. If the pod already got picked and
|
||||
// modified, this should fail; in that case just
|
||||
// retry.
|
||||
chosenPod.ObjectMeta.Labels = newLabels
|
||||
gp.logger.Info("relabeling pod", zap.String("pod", chosenPod.ObjectMeta.Name))
|
||||
_, err = gp.kubernetesClient.CoreV1().Pods(gp.namespace).Update(chosenPod)
|
||||
if err != nil {
|
||||
gp.logger.Error("failed to relabel pod", zap.Error(err), zap.String("pod", chosenPod.ObjectMeta.Name))
|
||||
continue
|
||||
}
|
||||
}
|
||||
gp.logger.Info("chose pod", zap.String("pod", chosenPod.ObjectMeta.Name), zap.Duration("elapsed_time", time.Since(startTime)))
|
||||
return chosenPod, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (gp *GenericPool) labelsForFunction(metadata *metav1.ObjectMeta) map[string]string {
|
||||
label := gp.getDeployLabels()
|
||||
label[types.FUNCTION_NAME] = metadata.Name
|
||||
label[types.FUNCTION_UID] = string(metadata.UID)
|
||||
label[types.FUNCTION_NAMESPACE] = metadata.Namespace // function CRD must stay within same namespace of environment CRD
|
||||
label["managed"] = "false" // this allows us to easily find pods not managed by the deployment
|
||||
return label
|
||||
|
||||
}
|
||||
|
||||
func (gp *GenericPool) scheduleDeletePod(name string) {
|
||||
go func() {
|
||||
// The sleep allows debugging or collecting logs from the pod before it's
|
||||
// cleaned up. (We need a better solutions for both those things; log
|
||||
// aggregation and storage will help.)
|
||||
gp.logger.Error("error in pod - scheduling cleanup", zap.String("pod", name))
|
||||
// Ignore sleep here if istio feature is enabled, function pod
|
||||
// will be deleted after 6 mins (terminationGracePeriodSeconds).
|
||||
if !gp.useIstio {
|
||||
time.Sleep(5 * time.Minute)
|
||||
}
|
||||
gp.kubernetesClient.CoreV1().Pods(gp.namespace).Delete(name, nil)
|
||||
}()
|
||||
}
|
||||
|
||||
func IsIPv6(podIP string) bool {
|
||||
ip := net.ParseIP(podIP)
|
||||
return ip != nil && strings.Contains(podIP, ":")
|
||||
}
|
||||
|
||||
func (gp *GenericPool) getSpecializeUrl(podIP string) string {
|
||||
testUrl := os.Getenv("TEST_SPECIALIZE_URL")
|
||||
if len(testUrl) != 0 {
|
||||
// it takes a second or so for the test service to
|
||||
// become routable once a pod is relabeled. This is
|
||||
// super hacky, but only runs in unit tests.
|
||||
time.Sleep(5 * time.Second)
|
||||
return testUrl
|
||||
}
|
||||
isv6 := IsIPv6(podIP)
|
||||
var baseUrl string
|
||||
if isv6 == false {
|
||||
baseUrl = fmt.Sprintf("http://%v:8000/", podIP)
|
||||
} else if isv6 == true { // We use bracket if the IP is in IPv6.
|
||||
baseUrl = fmt.Sprintf("http://[%v]:8000/", podIP)
|
||||
}
|
||||
return baseUrl
|
||||
|
||||
}
|
||||
|
||||
// specializePod chooses a pod, copies the required user-defined function to that pod
|
||||
// (via fetcher), and calls the function-run container to load it, resulting in a
|
||||
// specialized pod.
|
||||
func (gp *GenericPool) specializePod(ctx context.Context, pod *apiv1.Pod, metadata *metav1.ObjectMeta) error {
|
||||
// for fetcher we don't need to create a service, just talk to the pod directly
|
||||
podIP := pod.Status.PodIP
|
||||
if len(podIP) == 0 {
|
||||
return errors.Errorf("Pod %s in namespace %s has no IP", pod.ObjectMeta.Name, pod.ObjectMeta.Namespace)
|
||||
}
|
||||
// specialize pod with service
|
||||
if gp.useIstio {
|
||||
svc := utils.GetFunctionIstioServiceName(metadata.Name, metadata.Namespace)
|
||||
podIP = fmt.Sprintf("%v.%v", svc, gp.namespace)
|
||||
}
|
||||
|
||||
// tell fetcher to get the function.
|
||||
fetcherUrl := gp.getSpecializeUrl(podIP)
|
||||
gp.logger.Info("calling fetcher to copy function", zap.String("function", metadata.Name), zap.String("url", fetcherUrl))
|
||||
|
||||
fn, err := gp.fissionClient.
|
||||
Functions(metadata.Namespace).
|
||||
Get(metadata.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
specializeReq := gp.fetcherConfig.NewSpecializeRequest(fn, gp.env)
|
||||
|
||||
gp.logger.Info("specializing pod", zap.String("function", metadata.Name))
|
||||
|
||||
err = fetcherClient.MakeClient(gp.logger, fetcherUrl).Specialize(ctx, &specializeReq)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getPoolName returns a unique name of an environment
|
||||
func (gp *GenericPool) getPoolName() string {
|
||||
return strings.ToLower(fmt.Sprintf("poolmgr-%v-%v-%v", gp.env.Metadata.Name, gp.env.Metadata.Namespace, uniuri.NewLen(8)))
|
||||
}
|
||||
|
||||
// A pool is a deployment of generic containers for an env. This
|
||||
// creates the pool but doesn't wait for any pods to be ready.
|
||||
func (gp *GenericPool) createPool() error {
|
||||
// Use long terminationGracePeriodSeconds for connection draining in case that
|
||||
// pod still runs user functions.
|
||||
gracePeriodSeconds := int64(6 * 60)
|
||||
if gp.env.Spec.TerminationGracePeriod > 0 {
|
||||
gracePeriodSeconds = gp.env.Spec.TerminationGracePeriod
|
||||
}
|
||||
|
||||
podAnnotations := gp.env.Metadata.Annotations
|
||||
if podAnnotations == nil {
|
||||
podAnnotations = make(map[string]string)
|
||||
}
|
||||
if gp.useIstio && gp.env.Spec.AllowAccessToExternalNetwork {
|
||||
podAnnotations["sidecar.istio.io/inject"] = "false"
|
||||
}
|
||||
|
||||
deployment := &v1beta1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: gp.getPoolName(),
|
||||
Labels: gp.labelsForPool,
|
||||
},
|
||||
Spec: v1beta1.DeploymentSpec{
|
||||
Replicas: &gp.replicas,
|
||||
Selector: &metav1.LabelSelector{
|
||||
MatchLabels: gp.labelsForPool,
|
||||
},
|
||||
Template: apiv1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: gp.labelsForPool,
|
||||
Annotations: podAnnotations,
|
||||
},
|
||||
Spec: apiv1.PodSpec{
|
||||
Containers: []apiv1.Container{
|
||||
util.MergeContainerSpecs(&apiv1.Container{
|
||||
Name: gp.env.Metadata.Name,
|
||||
Image: gp.env.Spec.Runtime.Image,
|
||||
ImagePullPolicy: gp.runtimeImagePullPolicy,
|
||||
TerminationMessagePath: "/dev/termination-log",
|
||||
Resources: gp.env.Spec.Resources,
|
||||
// Pod is removed from endpoints list for service when it's
|
||||
// state became "Termination". We used preStop hook as the
|
||||
// workaround for connection draining since pod maybe shutdown
|
||||
// before grace period expires.
|
||||
// https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods
|
||||
// https://github.com/kubernetes/kubernetes/issues/47576#issuecomment-308900172
|
||||
Lifecycle: &apiv1.Lifecycle{
|
||||
PreStop: &apiv1.Handler{
|
||||
Exec: &apiv1.ExecAction{
|
||||
Command: []string{
|
||||
"/bin/sleep",
|
||||
fmt.Sprintf("%v", gracePeriodSeconds),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}, gp.env.Spec.Runtime.Container),
|
||||
},
|
||||
ServiceAccountName: "fission-fetcher",
|
||||
// TerminationGracePeriodSeconds should be equal to the
|
||||
// sleep time of preStop to make sure that SIGTERM is sent
|
||||
// to pod after 6 mins.
|
||||
TerminationGracePeriodSeconds: &gracePeriodSeconds,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Order of merging is important here - first fetcher, then containers and lastly pod spec
|
||||
err := gp.fetcherConfig.AddFetcherToPodSpec(&deployment.Spec.Template.Spec, gp.env.Metadata.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if gp.env.Spec.Runtime.PodSpec != nil {
|
||||
err = util.MergePodSpec(&deployment.Spec.Template.Spec, gp.env.Spec.Runtime.PodSpec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment)
|
||||
if err != nil {
|
||||
gp.logger.Error("error creating deployment in kubernetes", zap.Error(err), zap.String("deployment", deployment.Name))
|
||||
return err
|
||||
}
|
||||
gp.deployment = depl
|
||||
return nil
|
||||
}
|
||||
|
||||
func (gp *GenericPool) waitForReadyPod() error {
|
||||
startTime := time.Now()
|
||||
for {
|
||||
// TODO: for now we just poll; use a watch instead
|
||||
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Get(
|
||||
gp.deployment.ObjectMeta.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
e := "error waiting for ready pod for deployment"
|
||||
gp.logger.Error(e, zap.String("deployment", gp.deployment.ObjectMeta.Name), zap.String("namespace", gp.namespace))
|
||||
return fmt.Errorf("%s %q in namespace %q", e, gp.deployment.ObjectMeta.Name, gp.namespace)
|
||||
}
|
||||
|
||||
gp.deployment = depl
|
||||
if gp.deployment.Status.AvailableReplicas > 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if time.Since(startTime) > gp.podReadyTimeout {
|
||||
podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(metav1.ListOptions{
|
||||
LabelSelector: labels.Set(
|
||||
gp.deployment.Spec.Selector.MatchLabels).AsSelector().String(),
|
||||
})
|
||||
if err != nil {
|
||||
gp.logger.Error("error getting pod list after timeout waiting for ready pod", zap.Error(err))
|
||||
}
|
||||
|
||||
// Since even single pod is not ready, choosing the first pod to inspect is a good approximation. In future this can be done better
|
||||
pod := podList.Items[0]
|
||||
var multierr *multierror.Error
|
||||
for _, cStatus := range pod.Status.ContainerStatuses {
|
||||
if cStatus.Ready != true {
|
||||
multierr = multierror.Append(multierr, errors.New(fmt.Sprintf("%v: %v", cStatus.State.Waiting.Reason, cStatus.State.Waiting.Message)))
|
||||
}
|
||||
}
|
||||
return errors.Wrapf(multierr, "Timeout: waited too long for pod of deployment %v in namespace %v to be ready",
|
||||
gp.deployment.ObjectMeta.Name, gp.namespace)
|
||||
}
|
||||
time.Sleep(1000 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func (gp *GenericPool) createSvc(name string, labels map[string]string) (*apiv1.Service, error) {
|
||||
service := apiv1.Service{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: name,
|
||||
},
|
||||
Spec: apiv1.ServiceSpec{
|
||||
Type: apiv1.ServiceTypeClusterIP,
|
||||
Ports: []apiv1.ServicePort{
|
||||
{
|
||||
Protocol: apiv1.ProtocolTCP,
|
||||
Port: 80,
|
||||
TargetPort: intstr.FromInt(8888),
|
||||
},
|
||||
},
|
||||
Selector: labels,
|
||||
},
|
||||
}
|
||||
svc, err := gp.kubernetesClient.CoreV1().Services(gp.namespace).Create(&service)
|
||||
return svc, err
|
||||
}
|
||||
|
||||
func (gp *GenericPool) GetFuncSvc(ctx context.Context, m *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
|
||||
gp.logger.Info("choosing pod from pool", zap.String("function", m.Name))
|
||||
newLabels := gp.labelsForFunction(m)
|
||||
|
||||
if gp.useIstio {
|
||||
// Istio only allows accessing pod through k8s service, and requests come to
|
||||
// service are not always being routed to the same pod. For example:
|
||||
|
||||
// If there is only one pod (podA) behind the service svcX.
|
||||
|
||||
// svcX -> podA
|
||||
|
||||
// All requests (specialize request & function access requests)
|
||||
// will be routed to podA without any problem.
|
||||
|
||||
// If podA and podB are behind svcX.
|
||||
|
||||
// svcX -> podA (specialized)
|
||||
// -> podB (non-specialized)
|
||||
|
||||
// The specialize request may be routed to podA and the function access
|
||||
// requests may go to podB. In this case, the function cannot be served
|
||||
// properly.
|
||||
|
||||
// To prevent such problem, we need to delete old versions function pods
|
||||
// and make sure that there is only one pod behind the service
|
||||
|
||||
sel := map[string]string{
|
||||
"functionName": m.Name,
|
||||
"functionUid": string(m.UID),
|
||||
}
|
||||
podList, err := gp.kubernetesClient.CoreV1().Pods(gp.namespace).List(metav1.ListOptions{
|
||||
LabelSelector: labels.Set(sel).AsSelector().String(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Remove old versions function pods
|
||||
for _, pod := range podList.Items {
|
||||
// Delete pod no matter what status it is
|
||||
gp.kubernetesClient.CoreV1().Pods(gp.namespace).Delete(pod.ObjectMeta.Name, nil)
|
||||
}
|
||||
}
|
||||
|
||||
pod, err := gp.choosePod(newLabels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = gp.specializePod(ctx, pod, m)
|
||||
if err != nil {
|
||||
gp.scheduleDeletePod(pod.ObjectMeta.Name)
|
||||
return nil, err
|
||||
}
|
||||
gp.logger.Info("specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
|
||||
|
||||
var svcHost string
|
||||
if gp.useSvc && !gp.useIstio {
|
||||
svcName := fmt.Sprintf("svc-%v", m.Name)
|
||||
if len(m.UID) > 0 {
|
||||
svcName = fmt.Sprintf("%s-%v", svcName, m.UID)
|
||||
}
|
||||
|
||||
labels := gp.labelsForFunction(m)
|
||||
svc, err := gp.createSvc(svcName, labels)
|
||||
if err != nil {
|
||||
gp.scheduleDeletePod(pod.ObjectMeta.Name)
|
||||
return nil, err
|
||||
}
|
||||
if svc.ObjectMeta.Name != svcName {
|
||||
gp.scheduleDeletePod(pod.ObjectMeta.Name)
|
||||
return nil, errors.Errorf("sanity check failed for svc %v", svc.ObjectMeta.Name)
|
||||
}
|
||||
|
||||
// the fission router isn't in the same namespace, so return a
|
||||
// namespace-qualified hostname
|
||||
svcHost = fmt.Sprintf("%v.%v", svcName, gp.namespace)
|
||||
} else if gp.useIstio {
|
||||
svc := utils.GetFunctionIstioServiceName(m.Name, m.Namespace)
|
||||
svcHost = fmt.Sprintf("%v.%v:8888", svc, gp.namespace)
|
||||
} else {
|
||||
gp.logger.Info("using pod IP for specialized pod", zap.String("pod", pod.ObjectMeta.Name), zap.String("function", m.Name))
|
||||
svcHost = fmt.Sprintf("%v:8888", pod.Status.PodIP)
|
||||
}
|
||||
|
||||
kubeObjRefs := []apiv1.ObjectReference{
|
||||
{
|
||||
Kind: "pod",
|
||||
Name: pod.ObjectMeta.Name,
|
||||
APIVersion: pod.TypeMeta.APIVersion,
|
||||
Namespace: pod.ObjectMeta.Namespace,
|
||||
ResourceVersion: pod.ObjectMeta.ResourceVersion,
|
||||
UID: pod.ObjectMeta.UID,
|
||||
},
|
||||
}
|
||||
|
||||
fsvc := &fscache.FuncSvc{
|
||||
Name: pod.ObjectMeta.Name,
|
||||
Function: m,
|
||||
Environment: gp.env,
|
||||
Address: svcHost,
|
||||
KubernetesObjects: kubeObjRefs,
|
||||
Executor: fscache.POOLMGR,
|
||||
Ctime: time.Now(),
|
||||
Atime: time.Now(),
|
||||
}
|
||||
|
||||
_, err = gp.fsCache.Add(*fsvc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fsvc, nil
|
||||
}
|
||||
|
||||
// destroys the pool -- the deployment, replicaset and pods
|
||||
func (gp *GenericPool) destroy() error {
|
||||
deletePropagation := metav1.DeletePropagationBackground
|
||||
delOpt := metav1.DeleteOptions{
|
||||
PropagationPolicy: &deletePropagation,
|
||||
}
|
||||
err := gp.kubernetesClient.ExtensionsV1beta1().
|
||||
Deployments(gp.namespace).Delete(gp.deployment.ObjectMeta.Name, &delOpt)
|
||||
if err != nil {
|
||||
gp.logger.Error("error destroying deployment",
|
||||
zap.Error(err),
|
||||
zap.String("deployment_name", gp.deployment.ObjectMeta.Name),
|
||||
zap.String("deployment_namespace", gp.namespace))
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package poolmgr
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
"go.uber.org/zap"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
k8sTypes "k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/cache"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
"github.com/fission/fission/pkg/executor/fscache"
|
||||
"github.com/fission/fission/pkg/executor/reaper"
|
||||
fetcherConfig "github.com/fission/fission/pkg/fetcher/config"
|
||||
"github.com/fission/fission/pkg/types"
|
||||
)
|
||||
|
||||
type requestType int
|
||||
|
||||
const (
|
||||
GET_POOL requestType = iota
|
||||
CLEANUP_POOLS
|
||||
)
|
||||
|
||||
type (
|
||||
GenericPoolManager struct {
|
||||
logger *zap.Logger
|
||||
|
||||
pools map[string]*GenericPool
|
||||
kubernetesClient *kubernetes.Clientset
|
||||
namespace string
|
||||
|
||||
fissionClient *crd.FissionClient
|
||||
functionEnv *cache.Cache
|
||||
fsCache *fscache.FunctionServiceCache
|
||||
instanceId string
|
||||
requestChannel chan *request
|
||||
|
||||
enableIstio bool
|
||||
fetcherConfig *fetcherConfig.Config
|
||||
|
||||
funcStore k8sCache.Store
|
||||
funcController k8sCache.Controller
|
||||
pkgStore k8sCache.Store
|
||||
pkgController k8sCache.Controller
|
||||
|
||||
idlePodReapTime time.Duration
|
||||
}
|
||||
request struct {
|
||||
requestType
|
||||
env *fv1.Environment
|
||||
envList []fv1.Environment
|
||||
responseChannel chan *response
|
||||
}
|
||||
response struct {
|
||||
error
|
||||
pool *GenericPool
|
||||
}
|
||||
)
|
||||
|
||||
func MakeGenericPoolManager(
|
||||
logger *zap.Logger,
|
||||
fissionClient *crd.FissionClient,
|
||||
kubernetesClient *kubernetes.Clientset,
|
||||
functionNamespace string,
|
||||
fetcherConfig *fetcherConfig.Config,
|
||||
instanceId string) *GenericPoolManager {
|
||||
|
||||
gpmLogger := logger.Named("generic_pool_manager")
|
||||
|
||||
gpm := &GenericPoolManager{
|
||||
logger: gpmLogger,
|
||||
pools: make(map[string]*GenericPool),
|
||||
kubernetesClient: kubernetesClient,
|
||||
namespace: functionNamespace,
|
||||
fissionClient: fissionClient,
|
||||
functionEnv: cache.MakeCache(10*time.Second, 0),
|
||||
fsCache: fscache.MakeFunctionServiceCache(gpmLogger),
|
||||
instanceId: instanceId,
|
||||
requestChannel: make(chan *request),
|
||||
idlePodReapTime: 2 * time.Minute,
|
||||
fetcherConfig: fetcherConfig,
|
||||
}
|
||||
go gpm.service()
|
||||
go gpm.eagerPoolCreator()
|
||||
|
||||
if len(os.Getenv("ENABLE_ISTIO")) > 0 {
|
||||
istio, err := strconv.ParseBool(os.Getenv("ENABLE_ISTIO"))
|
||||
if err != nil {
|
||||
gpmLogger.Info("failed to parse ENABLE_ISTIO")
|
||||
}
|
||||
gpm.enableIstio = istio
|
||||
}
|
||||
|
||||
gpm.funcStore, gpm.funcController = gpm.makeFuncController(
|
||||
gpm.fissionClient, gpm.kubernetesClient, gpm.namespace, gpm.enableIstio)
|
||||
|
||||
gpm.pkgStore, gpm.pkgController = gpm.makePkgController(gpm.fissionClient, gpm.kubernetesClient, gpm.namespace)
|
||||
|
||||
return gpm
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) Run(ctx context.Context) {
|
||||
go gpm.funcController.Run(ctx.Done())
|
||||
go gpm.pkgController.Run(ctx.Done())
|
||||
go gpm.idleObjectReaper()
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) service() {
|
||||
for {
|
||||
req := <-gpm.requestChannel
|
||||
switch req.requestType {
|
||||
case GET_POOL:
|
||||
// just because they are missing in the cache, we end up creating another duplicate pool.
|
||||
var err error
|
||||
pool, ok := gpm.pools[crd.CacheKey(&req.env.Metadata)]
|
||||
if !ok {
|
||||
poolsize := gpm.getEnvPoolsize(req.env)
|
||||
switch req.env.Spec.AllowedFunctionsPerContainer {
|
||||
case types.AllowedFunctionsPerContainerInfinite:
|
||||
poolsize = 1
|
||||
}
|
||||
|
||||
// To support backward compatibility, if envs are created in default ns, we go ahead
|
||||
// and create pools in fission-function ns as earlier.
|
||||
ns := gpm.namespace
|
||||
if req.env.Metadata.Namespace != metav1.NamespaceDefault {
|
||||
ns = req.env.Metadata.Namespace
|
||||
}
|
||||
|
||||
pool, err = MakeGenericPool(gpm.logger,
|
||||
gpm.fissionClient, gpm.kubernetesClient, req.env, poolsize,
|
||||
ns, gpm.namespace, gpm.fsCache, gpm.fetcherConfig, gpm.instanceId, gpm.enableIstio)
|
||||
if err != nil {
|
||||
req.responseChannel <- &response{error: err}
|
||||
continue
|
||||
}
|
||||
gpm.pools[crd.CacheKey(&req.env.Metadata)] = pool
|
||||
}
|
||||
req.responseChannel <- &response{pool: pool}
|
||||
case CLEANUP_POOLS:
|
||||
latestEnvPoolsize := make(map[string]int)
|
||||
for _, env := range req.envList {
|
||||
latestEnvPoolsize[crd.CacheKey(&env.Metadata)] = int(gpm.getEnvPoolsize(&env))
|
||||
}
|
||||
for key, pool := range gpm.pools {
|
||||
poolsize, ok := latestEnvPoolsize[key]
|
||||
if !ok || poolsize == 0 {
|
||||
// Env no longer exists or pool size changed to zero
|
||||
|
||||
gpm.logger.Info("destroying generic pool", zap.Any("environment", pool.env.Metadata))
|
||||
delete(gpm.pools, key)
|
||||
|
||||
// and delete the pool asynchronously.
|
||||
go pool.destroy()
|
||||
}
|
||||
}
|
||||
// no response, caller doesn't wait
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) GetPool(env *fv1.Environment) (*GenericPool, error) {
|
||||
c := make(chan *response)
|
||||
gpm.requestChannel <- &request{
|
||||
requestType: GET_POOL,
|
||||
env: env,
|
||||
responseChannel: c,
|
||||
}
|
||||
resp := <-c
|
||||
return resp.pool, resp.error
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) CleanupPools(envs []fv1.Environment) {
|
||||
gpm.requestChannel <- &request{
|
||||
requestType: CLEANUP_POOLS,
|
||||
envList: envs,
|
||||
}
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) GetFuncSvc(ctx context.Context, metadata *metav1.ObjectMeta) (*fscache.FuncSvc, error) {
|
||||
// from Func -> get Env
|
||||
gpm.logger.Info("getting environment for function", zap.String("function", metadata.Name))
|
||||
env, err := gpm.getFunctionEnv(metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pool, err := gpm.GetPool(env)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// from GenericPool -> get one function container
|
||||
// (this also adds to the cache)
|
||||
gpm.logger.Info("getting function service from pool", zap.String("function", metadata.Name))
|
||||
return pool.GetFuncSvc(ctx, metadata)
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) getFunctionEnv(m *metav1.ObjectMeta) (*fv1.Environment, error) {
|
||||
var env *fv1.Environment
|
||||
|
||||
// Cached ?
|
||||
result, err := gpm.functionEnv.Get(crd.CacheKey(m))
|
||||
if err == nil {
|
||||
env = result.(*fv1.Environment)
|
||||
return env, nil
|
||||
}
|
||||
|
||||
// Cache miss -- get func from controller
|
||||
f, err := gpm.fissionClient.Functions(m.Namespace).Get(m.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get env from metadata
|
||||
gpm.logger.Info("getting env", zap.Any("function", m))
|
||||
env, err = gpm.fissionClient.Environments(f.Spec.Environment.Namespace).Get(f.Spec.Environment.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// cache for future lookups
|
||||
gpm.functionEnv.Set(crd.CacheKey(m), env)
|
||||
|
||||
return env, nil
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) eagerPoolCreator() {
|
||||
pollSleep := time.Duration(2 * time.Second)
|
||||
for {
|
||||
// get list of envs from controller
|
||||
envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
if utils.IsNetworkError(err) {
|
||||
gpm.logger.Error("encountered network error, retrying", zap.Error(err))
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
gpm.logger.Fatal("failed to get environment list", zap.Error(err))
|
||||
}
|
||||
|
||||
// Create pools for all envs. TODO: we should make this a bit less eager, only
|
||||
// creating pools for envs that are actually used by functions. Also we might want
|
||||
// to keep these eagerly created pools smaller than the ones created when there are
|
||||
// actual function calls.
|
||||
for i := range envs.Items {
|
||||
env := envs.Items[i]
|
||||
// Create pool only if poolsize greater than zero
|
||||
if gpm.getEnvPoolsize(&env) > 0 {
|
||||
_, err := gpm.GetPool(&envs.Items[i])
|
||||
if err != nil {
|
||||
gpm.logger.Error("eager-create pool failed", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up pools whose env was deleted
|
||||
gpm.CleanupPools(envs.Items)
|
||||
time.Sleep(pollSleep)
|
||||
}
|
||||
}
|
||||
|
||||
func (gpm *GenericPoolManager) getEnvPoolsize(env *fv1.Environment) int32 {
|
||||
var poolsize int32
|
||||
if env.Spec.Version < 3 {
|
||||
poolsize = 3
|
||||
} else {
|
||||
poolsize = int32(env.Spec.Poolsize)
|
||||
}
|
||||
return poolsize
|
||||
}
|
||||
|
||||
// IsValid checks if pod is not deleted and that it has the address passed as the argument. Also checks that all the
|
||||
// containers in it are reporting a ready status for the healthCheck.
|
||||
func (gpm *GenericPoolManager) IsValid(fsvc *fscache.FuncSvc) bool {
|
||||
for _, obj := range fsvc.KubernetesObjects {
|
||||
if obj.Kind == "pod" {
|
||||
pod, err := gpm.kubernetesClient.CoreV1().Pods(obj.Namespace).Get(obj.Name, metav1.GetOptions{})
|
||||
if err == nil && strings.Contains(fsvc.Address, pod.Status.PodIP) && utils.IsReadyPod(pod) {
|
||||
gpm.logger.Info("valid pod address", zap.String("address", fsvc.Address))
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// idleObjectReaper reaps objects after certain idle time
|
||||
func (gpm *GenericPoolManager) idleObjectReaper() {
|
||||
|
||||
pollSleep := time.Duration(gpm.idlePodReapTime)
|
||||
for {
|
||||
time.Sleep(pollSleep)
|
||||
|
||||
envs, err := gpm.fissionClient.Environments(metav1.NamespaceAll).List(metav1.ListOptions{})
|
||||
if err != nil {
|
||||
gpm.logger.Fatal("failed to get environment list", zap.Error(err))
|
||||
}
|
||||
|
||||
envList := make(map[k8sTypes.UID]struct{})
|
||||
for _, env := range envs.Items {
|
||||
envList[env.Metadata.UID] = struct{}{}
|
||||
}
|
||||
|
||||
funcSvcs, err := gpm.fsCache.ListOld(gpm.idlePodReapTime)
|
||||
if err != nil {
|
||||
gpm.logger.Error("error reaping idle pods", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
for _, fsvc := range funcSvcs {
|
||||
if fsvc.Executor != fscache.POOLMGR {
|
||||
continue
|
||||
}
|
||||
|
||||
// For function with the environment that no longer exists, executor
|
||||
// cleanups the idle pod as usual and prints log to notify user.
|
||||
if _, ok := envList[fsvc.Environment.Metadata.UID]; !ok {
|
||||
gpm.logger.Info("function environment no longer exists",
|
||||
zap.String("environment", fsvc.Environment.Metadata.Name),
|
||||
zap.String("function", fsvc.Name))
|
||||
}
|
||||
|
||||
if fsvc.Environment.Spec.AllowedFunctionsPerContainer == types.AllowedFunctionsPerContainerInfinite {
|
||||
continue
|
||||
}
|
||||
|
||||
deleted, err := gpm.fsCache.DeleteOld(fsvc, gpm.idlePodReapTime)
|
||||
if err != nil {
|
||||
gpm.logger.Error("error deleting Kubernetes objects for function service",
|
||||
zap.Error(err),
|
||||
zap.Any("service", fsvc))
|
||||
}
|
||||
|
||||
if !deleted {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, kubeobj := range fsvc.KubernetesObjects {
|
||||
reaper.CleanupKubeObject(gpm.logger, gpm.kubernetesClient, &kubeobj)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
Copyright 2018 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package poolmgr
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/fission/fission/pkg/types"
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
"go.uber.org/zap"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
k8sCache "k8s.io/client-go/tools/cache"
|
||||
|
||||
fv1 "github.com/fission/fission/pkg/apis/fission.io/v1"
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
)
|
||||
|
||||
// TODO : It may make sense to make each of add, update, delete funcs run as separate go routines.
|
||||
func (gpm *GenericPoolManager) makePkgController(fissionClient *crd.FissionClient,
|
||||
kubernetesClient *kubernetes.Clientset, fissionfnNamespace string) (k8sCache.Store, k8sCache.Controller) {
|
||||
|
||||
resyncPeriod := 30 * time.Second
|
||||
lw := k8sCache.NewListWatchFromClient(fissionClient.GetCrdClient(), "packages", metav1.NamespaceAll, fields.Everything())
|
||||
pkgStore, controller := k8sCache.NewInformer(lw, &fv1.Package{}, resyncPeriod,
|
||||
k8sCache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(obj interface{}) {
|
||||
pkg := obj.(*fv1.Package)
|
||||
gpm.logger.Debug("list watch for package reported a new package addition",
|
||||
zap.String("package_name", pkg.Metadata.Name),
|
||||
zap.String("package_namepsace", pkg.Metadata.Namespace))
|
||||
|
||||
// create or update role-binding for fetcher sa in env ns to be able to get the pkg contents from pkg namespace
|
||||
envNs := fissionfnNamespace
|
||||
if pkg.Spec.Environment.Namespace != metav1.NamespaceDefault {
|
||||
envNs = pkg.Spec.Environment.Namespace
|
||||
}
|
||||
|
||||
// here, we return if we hit an error during rolebinding setup. this is because this rolebinding is mandatory for
|
||||
// every function's package to be loaded into its env. without that, there's no point to move forward.
|
||||
err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.PackageGetterRB, pkg.Metadata.Namespace, types.PackageGetterCR, types.ClusterRole, types.FissionFetcherSA, envNs)
|
||||
if err != nil {
|
||||
gpm.logger.Error("error creating rolebinding for package",
|
||||
zap.Error(err),
|
||||
zap.String("role_binding", types.PackageGetterRB),
|
||||
zap.String("package_name", pkg.Metadata.Name),
|
||||
zap.String("package_namespace", pkg.Metadata.Namespace))
|
||||
return
|
||||
}
|
||||
|
||||
gpm.logger.Debug("successfully set up rolebinding for fetcher service account",
|
||||
zap.String("service_account", types.FissionFetcherSA),
|
||||
zap.String("service_account_namespace", envNs),
|
||||
zap.String("package_name", pkg.Metadata.Name),
|
||||
zap.String("package_namespace", pkg.Metadata.Namespace))
|
||||
},
|
||||
|
||||
UpdateFunc: func(oldObj, newObj interface{}) {
|
||||
oldPkg := oldObj.(*fv1.Package)
|
||||
newPkg := newObj.(*fv1.Package)
|
||||
|
||||
if oldPkg.Metadata.ResourceVersion == newPkg.Metadata.ResourceVersion {
|
||||
return
|
||||
}
|
||||
|
||||
// if a pkg's env reference gets updated and the newly referenced env is in a different ns,
|
||||
// we need to update the role-binding in pkg ns to grant permissions to the fetcher-sa in env ns
|
||||
// to do a get on pkg
|
||||
if oldPkg.Spec.Environment.Namespace != newPkg.Spec.Environment.Namespace {
|
||||
envNs := fissionfnNamespace
|
||||
if newPkg.Spec.Environment.Namespace != metav1.NamespaceDefault {
|
||||
envNs = newPkg.Spec.Environment.Namespace
|
||||
}
|
||||
|
||||
err := utils.SetupRoleBinding(gpm.logger, kubernetesClient, types.PackageGetterRB,
|
||||
newPkg.Metadata.Namespace, types.PackageGetterCR, types.ClusterRole,
|
||||
types.FissionFetcherSA, envNs)
|
||||
if err != nil {
|
||||
gpm.logger.Error("error updating rolebinding for package",
|
||||
zap.Error(err),
|
||||
zap.String("role_binding", types.PackageGetterRB),
|
||||
zap.String("package_name", newPkg.Metadata.Name),
|
||||
zap.String("package_namespace", newPkg.Metadata.Namespace))
|
||||
return
|
||||
}
|
||||
|
||||
gpm.logger.Debug("successfully updated rolebinding for fetcher service account",
|
||||
zap.String("service_account", types.FissionFetcherSA),
|
||||
zap.String("service_account_namespace", envNs),
|
||||
zap.String("package_name", newPkg.Metadata.Name),
|
||||
zap.String("package_namespace", newPkg.Metadata.Namespace))
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
return pkgStore, controller
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package reaper
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/fission/fission/pkg/utils"
|
||||
"go.uber.org/zap"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
"github.com/fission/fission/pkg/crd"
|
||||
"github.com/fission/fission/pkg/types"
|
||||
)
|
||||
|
||||
var (
|
||||
deletePropagation = meta_v1.DeletePropagationBackground
|
||||
delOpt = meta_v1.DeleteOptions{PropagationPolicy: &deletePropagation}
|
||||
)
|
||||
|
||||
// CleanupOldExecutorObjects cleans up resources created by old executor instances
|
||||
func CleanupOldExecutorObjects(logger *zap.Logger, kubernetesClient *kubernetes.Clientset, instanceId string) {
|
||||
go func() {
|
||||
err := cleanup(logger, kubernetesClient, instanceId)
|
||||
if err != nil {
|
||||
// TODO retry reaper; logged and ignored for now
|
||||
logger.Error("Failed to cleanup old executor objects", zap.Error(err))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func cleanup(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
|
||||
|
||||
err := cleanupServices(logger, client, instanceId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = cleanupHpa(logger, client, instanceId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Deployments are used for idle pools and can be cleaned up
|
||||
// immediately. (We should "adopt" these instead of creating
|
||||
// a new pool.)
|
||||
err = cleanupDeployments(logger, client, instanceId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Pods might still be running user functions, so we give them
|
||||
// a few minutes before terminating them. This time is the
|
||||
// maximum function runtime, plus the time a router might
|
||||
// still route to an old instance, i.e. router cache expiry
|
||||
// time.
|
||||
time.Sleep(6 * time.Minute)
|
||||
|
||||
err = cleanupPods(logger, client, instanceId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupKubeObject deletes given kubernetes object
|
||||
func CleanupKubeObject(logger *zap.Logger, kubeClient *kubernetes.Clientset, kubeobj *apiv1.ObjectReference) {
|
||||
switch strings.ToLower(kubeobj.Kind) {
|
||||
case "pod":
|
||||
err := kubeClient.CoreV1().Pods(kubeobj.Namespace).Delete(kubeobj.Name, nil)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up pod", zap.Error(err), zap.String("pod", kubeobj.Name))
|
||||
}
|
||||
|
||||
case "service":
|
||||
err := kubeClient.CoreV1().Services(kubeobj.Namespace).Delete(kubeobj.Name, nil)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up service", zap.Error(err), zap.String("service", kubeobj.Name))
|
||||
}
|
||||
|
||||
case "deployment":
|
||||
err := kubeClient.ExtensionsV1beta1().Deployments(kubeobj.Namespace).Delete(kubeobj.Name, &delOpt)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up deployment", zap.Error(err), zap.String("deployment", kubeobj.Name))
|
||||
}
|
||||
|
||||
case "horizontalpodautoscaler":
|
||||
err := kubeClient.AutoscalingV1().HorizontalPodAutoscalers(kubeobj.Namespace).Delete(kubeobj.Name, nil)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up horizontalpodautoscaler", zap.Error(err), zap.String("horizontalpodautoscaler", kubeobj.Name))
|
||||
}
|
||||
|
||||
default:
|
||||
logger.Error("Could not identifying the object type to clean up", zap.String("type", kubeobj.Kind), zap.Any("object", kubeobj))
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupDeployments(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
|
||||
deploymentList, err := client.ExtensionsV1beta1().Deployments(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, dep := range deploymentList.Items {
|
||||
id, ok := dep.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
|
||||
if ok && id != instanceId {
|
||||
logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
|
||||
err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up deployment",
|
||||
zap.Error(err),
|
||||
zap.String("deployment_name", dep.ObjectMeta.Name),
|
||||
zap.String("deployment_namespace", dep.ObjectMeta.Namespace))
|
||||
}
|
||||
// ignore err
|
||||
}
|
||||
// Backward compatibility with older label name
|
||||
pid, pok := dep.ObjectMeta.Labels[types.POOLMGR_INSTANCEID_LABEL]
|
||||
if pok && pid != instanceId {
|
||||
logger.Info("cleaning up deployment", zap.String("deployment", dep.ObjectMeta.Name))
|
||||
err := client.ExtensionsV1beta1().Deployments(dep.ObjectMeta.Namespace).Delete(dep.ObjectMeta.Name, &delOpt)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up deployment",
|
||||
zap.Error(err),
|
||||
zap.String("deployment_name", dep.ObjectMeta.Name),
|
||||
zap.String("deployment_namespace", dep.ObjectMeta.Namespace))
|
||||
}
|
||||
// ignore err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupPods(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
|
||||
podList, err := client.CoreV1().Pods(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, pod := range podList.Items {
|
||||
id, ok := pod.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
|
||||
if ok && id != instanceId {
|
||||
logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
|
||||
err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up pod",
|
||||
zap.Error(err),
|
||||
zap.String("pod_name", pod.ObjectMeta.Name),
|
||||
zap.String("pod_namespace", pod.ObjectMeta.Namespace))
|
||||
}
|
||||
// ignore err
|
||||
}
|
||||
// Backward compatibility with older label name
|
||||
pid, pok := pod.ObjectMeta.Labels[types.POOLMGR_INSTANCEID_LABEL]
|
||||
if pok && pid != instanceId {
|
||||
logger.Info("cleaning up pod", zap.String("pod", pod.ObjectMeta.Name))
|
||||
err := client.CoreV1().Pods(pod.ObjectMeta.Namespace).Delete(pod.ObjectMeta.Name, nil)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up pod",
|
||||
zap.Error(err),
|
||||
zap.String("pod_name", pod.ObjectMeta.Name),
|
||||
zap.String("pod_namespace", pod.ObjectMeta.Namespace))
|
||||
}
|
||||
// ignore err
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupServices(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
|
||||
svcList, err := client.CoreV1().Services(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, svc := range svcList.Items {
|
||||
id, ok := svc.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
|
||||
if ok && id != instanceId {
|
||||
logger.Info("cleaning up service", zap.String("service", svc.ObjectMeta.Name))
|
||||
err := client.CoreV1().Services(svc.ObjectMeta.Namespace).Delete(svc.ObjectMeta.Name, nil)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up service",
|
||||
zap.Error(err),
|
||||
zap.String("service_name", svc.ObjectMeta.Name),
|
||||
zap.String("service_namespace", svc.ObjectMeta.Namespace))
|
||||
}
|
||||
// ignore err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupHpa(logger *zap.Logger, client *kubernetes.Clientset, instanceId string) error {
|
||||
hpaList, err := client.AutoscalingV1().HorizontalPodAutoscalers(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, hpa := range hpaList.Items {
|
||||
id, ok := hpa.ObjectMeta.Labels[types.EXECUTOR_INSTANCEID_LABEL]
|
||||
if ok && id != instanceId {
|
||||
logger.Info("cleaning up HPA", zap.String("hpa", hpa.ObjectMeta.Name))
|
||||
err := client.AutoscalingV1().HorizontalPodAutoscalers(hpa.ObjectMeta.Namespace).Delete(hpa.ObjectMeta.Name, nil)
|
||||
if err != nil {
|
||||
logger.Error("error cleaning up HPA",
|
||||
zap.Error(err),
|
||||
zap.String("hpa_name", hpa.ObjectMeta.Name),
|
||||
zap.String("hpa_namespace", hpa.ObjectMeta.Namespace))
|
||||
}
|
||||
// ignore err
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// CleanupRoleBindings periodically lists rolebindings across all namespaces and removes Service Accounts from them or
|
||||
// deletes the rolebindings completely if there are no Service Accounts in a rolebinding object.
|
||||
func CleanupRoleBindings(logger *zap.Logger, client *kubernetes.Clientset, fissionClient *crd.FissionClient, functionNs, envBuilderNs string, cleanupRoleBindingInterval time.Duration) {
|
||||
for {
|
||||
logger.Info("starting cleanupRoleBindings cycle")
|
||||
// get all rolebindings ( just to be efficient, one call to kubernetes )
|
||||
rbList, err := client.RbacV1beta1().RoleBindings(meta_v1.NamespaceAll).List(meta_v1.ListOptions{})
|
||||
if err != nil {
|
||||
// something wrong, but next iteration hopefully succeeds
|
||||
logger.Error("error listing role bindings in all namespaces", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
|
||||
// go through each role-binding object and do the cleanup necessary
|
||||
for _, roleBinding := range rbList.Items {
|
||||
// ignore role-bindings in kube-system namespace
|
||||
if roleBinding.Namespace == "kube-system" {
|
||||
continue
|
||||
}
|
||||
|
||||
// ignore role-bindings not created by fission
|
||||
if roleBinding.Name != types.PackageGetterRB && roleBinding.Name != types.SecretConfigMapGetterRB {
|
||||
continue
|
||||
}
|
||||
|
||||
// in order to find out if there are any functions that need this role-binding in role-binding namespace,
|
||||
// we can list the functions once per role-binding.
|
||||
funcList, err := fissionClient.Functions(roleBinding.Namespace).List(meta_v1.ListOptions{})
|
||||
if err != nil {
|
||||
logger.Error("error fetching environment list in namespace", zap.Error(err), zap.String("namespace", roleBinding.Namespace))
|
||||
continue
|
||||
}
|
||||
|
||||
// final map of service accounts that can be removed from this roleBinding object
|
||||
// using a map here instead of a list so the code in RemoveSAFromRoleBindingWithRetries is efficient.
|
||||
saToRemove := make(map[string]bool)
|
||||
|
||||
// the following flags are needed to decide if any of the service accounts can be removed from role-bindings depending on the functions that need them.
|
||||
// ndmFunc denotes if there's at least one function that has executor type New deploy Manager
|
||||
// funcEnvReference denotes if there's at least one function that has reference to an environment in the SA Namespace for the SA in question
|
||||
var ndmFunc, funcEnvReference bool
|
||||
|
||||
// iterate through each subject in the role-binding and check if there are any references to them
|
||||
for _, subj := range roleBinding.Subjects {
|
||||
ndmFunc = false
|
||||
funcEnvReference = false
|
||||
|
||||
// this is the reverse of what we're doing in setting up of role-bindings. if objects are created in default ns,
|
||||
// the SA namespace will have the value of "fission-function"/"fission-builder" depending on the SA.
|
||||
// so now we need to look for the objects in default namespace.
|
||||
saNs := subj.Namespace
|
||||
if subj.Namespace == functionNs ||
|
||||
subj.Namespace == envBuilderNs {
|
||||
saNs = meta_v1.NamespaceDefault
|
||||
}
|
||||
|
||||
// go through each function and find out if there's either at least one function with env reference in the same namespace as the Service Account in this iteration
|
||||
// or at least one function using ndm executor in the role-binding namespace and set the corresponding flags
|
||||
for _, fn := range funcList.Items {
|
||||
if fn.Spec.Environment.Namespace == saNs {
|
||||
funcEnvReference = true
|
||||
break
|
||||
}
|
||||
|
||||
if fn.Spec.InvokeStrategy.ExecutionStrategy.ExecutorType == types.ExecutorTypeNewdeploy {
|
||||
ndmFunc = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// if its a package-getterr-rb, we have 2 kinds of SAs and each of them is handled differently
|
||||
// else if its a secret-configmap-rb, we have only one SA which is fission-fetcher
|
||||
if roleBinding.Name == types.PackageGetterRB {
|
||||
// check if there is an env obj in saNs
|
||||
envList, err := fissionClient.Environments(saNs).List(meta_v1.ListOptions{})
|
||||
if err != nil {
|
||||
logger.Error("error fetching environment list in service account namespace", zap.Error(err), zap.String("namespace", saNs))
|
||||
continue
|
||||
}
|
||||
|
||||
// if the SA in this iteration is fission-builder, then we need to only check
|
||||
// if either there's at least one env object in the SA's namespace, or,
|
||||
// if there's at least one function in the role-binding namespace with env reference
|
||||
// to the SA's namespace.
|
||||
// if neither, then we can remove this SA from this role-binding
|
||||
if subj.Name == types.FissionBuilderSA {
|
||||
if len(envList.Items) == 0 && !funcEnvReference {
|
||||
saToRemove[utils.MakeSAMapKey(subj.Name, subj.Namespace)] = true
|
||||
}
|
||||
}
|
||||
|
||||
// if the SA in this iteration is fission-fetcher, then in addition to above checks,
|
||||
// we also need to check if there's at least one function with executor type New deploy
|
||||
// in the rolebinding's namespace.
|
||||
// if none of them are true, then remove this SA from this role-binding
|
||||
if subj.Name == types.FissionFetcherSA {
|
||||
if len(envList.Items) == 0 && !ndmFunc && !funcEnvReference {
|
||||
// remove SA from rolebinding
|
||||
saToRemove[utils.MakeSAMapKey(subj.Name, subj.Namespace)] = true
|
||||
}
|
||||
}
|
||||
} else if roleBinding.Name == types.SecretConfigMapGetterRB {
|
||||
// if there's not even one function in the role-binding's namespace and there's not even
|
||||
// one function with env reference to the SA's namespace, then remove that SA
|
||||
// from this role-binding
|
||||
if !ndmFunc && !funcEnvReference {
|
||||
saToRemove[utils.MakeSAMapKey(subj.Name, subj.Namespace)] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// finally, make a call to RemoveSAFromRoleBindingWithRetries for all the service accounts that need to be removed
|
||||
// for the role-binding in this iteration
|
||||
if len(saToRemove) != 0 {
|
||||
logger.Debug("removing service accounts from role binding",
|
||||
zap.Any("service_accounts", saToRemove),
|
||||
zap.String("role_binding_name", roleBinding.Name),
|
||||
zap.String("role_binding_namespace", roleBinding.Namespace))
|
||||
|
||||
// call this once in the end for each role-binding
|
||||
err = utils.RemoveSAFromRoleBindingWithRetries(logger, client, roleBinding.Name, roleBinding.Namespace, saToRemove)
|
||||
if err != nil {
|
||||
// if there's an error, we just log it and proceed with the next role-binding, hoping that this role-binding
|
||||
// will be processed in next iteration.
|
||||
logger.Debug("error removing service account from role binding",
|
||||
zap.Error(err),
|
||||
zap.Any("service_accounts", saToRemove),
|
||||
zap.String("role_binding_name", roleBinding.Name),
|
||||
zap.String("role_binding_namespace", roleBinding.Namespace))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// some sleep before the next reaper iteration
|
||||
time.Sleep(cleanupRoleBindingInterval)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package util
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/imdario/mergo"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
// MergeContainerSpecs merges container specs using a predefined order.
|
||||
//
|
||||
// The order of the arguments indicates which spec has precedence (lower index takes precedence over higher indexes).
|
||||
// Slices and maps are merged; other fields are set only if they are a zero value.
|
||||
func MergeContainerSpecs(specs ...*apiv1.Container) apiv1.Container {
|
||||
result := &apiv1.Container{}
|
||||
for _, spec := range specs {
|
||||
if spec == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
err := mergo.Merge(result, spec)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
return *result
|
||||
}
|
||||
|
||||
// mergeContainer is a specialized implementation of MergeContainerSpecs
|
||||
func mergeContainer(deployContainer *apiv1.Container, containerSpec apiv1.Container) error {
|
||||
|
||||
if &containerSpec == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if deployContainer.Name == containerSpec.Name {
|
||||
volMap := make(map[string]*apiv1.VolumeMount)
|
||||
for _, vol := range deployContainer.VolumeMounts {
|
||||
volMap[vol.Name] = &vol
|
||||
}
|
||||
for _, specVol := range containerSpec.VolumeMounts {
|
||||
_, ok := volMap[specVol.Name]
|
||||
if ok {
|
||||
return errors.New("Duplicate volume name found in the spec")
|
||||
} else {
|
||||
deployContainer.VolumeMounts = append(deployContainer.VolumeMounts, specVol)
|
||||
}
|
||||
}
|
||||
deployContainer.Env = append(deployContainer.Env, containerSpec.Env...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func MergePodSpec(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
|
||||
if &targetPodSpec == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var multierr *multierror.Error
|
||||
|
||||
// Get item from spec, if they exist in deployment - merge, else append
|
||||
// Same pattern for all lists (Mergo can not handle lists)
|
||||
// At some point this is better done with generics/reflection?
|
||||
err := mergeContainerLists(srcPodSpec, targetPodSpec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = mergeInitContainerList(srcPodSpec, targetPodSpec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// For volumes - if duplicate exist, throw error
|
||||
err = mergeVolumeLists(srcPodSpec, targetPodSpec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if targetPodSpec.NodeName != "" {
|
||||
srcPodSpec.NodeName = targetPodSpec.NodeName
|
||||
}
|
||||
|
||||
if targetPodSpec.Subdomain != "" {
|
||||
srcPodSpec.Subdomain = targetPodSpec.Subdomain
|
||||
}
|
||||
|
||||
if targetPodSpec.SchedulerName != "" {
|
||||
srcPodSpec.SchedulerName = targetPodSpec.SchedulerName
|
||||
}
|
||||
|
||||
if targetPodSpec.PriorityClassName != "" {
|
||||
srcPodSpec.PriorityClassName = targetPodSpec.PriorityClassName
|
||||
}
|
||||
|
||||
if targetPodSpec.TerminationGracePeriodSeconds != nil {
|
||||
srcPodSpec.TerminationGracePeriodSeconds = targetPodSpec.TerminationGracePeriodSeconds
|
||||
}
|
||||
|
||||
//TODO - Security context should be merged instead of overriding.
|
||||
if targetPodSpec.SecurityContext != nil {
|
||||
srcPodSpec.SecurityContext = targetPodSpec.SecurityContext
|
||||
}
|
||||
|
||||
//TODO - Affinity should be merged instead of overriding.
|
||||
if targetPodSpec.Affinity != nil {
|
||||
srcPodSpec.Affinity = targetPodSpec.Affinity
|
||||
}
|
||||
|
||||
if targetPodSpec.Hostname != "" {
|
||||
srcPodSpec.Hostname = targetPodSpec.Hostname
|
||||
}
|
||||
|
||||
for _, obj := range targetPodSpec.ImagePullSecrets {
|
||||
srcPodSpec.ImagePullSecrets = append(srcPodSpec.ImagePullSecrets, obj)
|
||||
}
|
||||
|
||||
for _, obj := range targetPodSpec.Tolerations {
|
||||
srcPodSpec.Tolerations = append(srcPodSpec.Tolerations, obj)
|
||||
}
|
||||
|
||||
for _, obj := range targetPodSpec.HostAliases {
|
||||
srcPodSpec.HostAliases = append(srcPodSpec.HostAliases, obj)
|
||||
}
|
||||
|
||||
err = mergo.Merge(&srcPodSpec.NodeSelector, targetPodSpec.NodeSelector)
|
||||
if err != nil {
|
||||
multierr = multierror.Append(multierr, err)
|
||||
}
|
||||
|
||||
return multierr.ErrorOrNil()
|
||||
}
|
||||
|
||||
func mergeContainerLists(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
|
||||
targetSpecContainers := targetPodSpec.Containers
|
||||
targetContainers := make(map[string]apiv1.Container)
|
||||
for _, c := range targetSpecContainers {
|
||||
targetContainers[c.Name] = c
|
||||
}
|
||||
|
||||
var multierr *multierror.Error
|
||||
for _, c := range srcPodSpec.Containers {
|
||||
container, ok := targetContainers[c.Name]
|
||||
if ok {
|
||||
err := mergeContainer(&c, container)
|
||||
multierr = multierror.Append(multierr, err)
|
||||
delete(targetContainers, c.Name)
|
||||
}
|
||||
}
|
||||
|
||||
for _, container := range targetContainers {
|
||||
srcPodSpec.Containers = append(srcPodSpec.Containers, container)
|
||||
}
|
||||
|
||||
return multierr.ErrorOrNil()
|
||||
}
|
||||
|
||||
func mergeInitContainerList(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
|
||||
targetSpecContainers := targetPodSpec.InitContainers
|
||||
targetContainers := make(map[string]apiv1.Container)
|
||||
for _, c := range targetSpecContainers {
|
||||
targetContainers[c.Name] = c
|
||||
}
|
||||
|
||||
var multierr *multierror.Error
|
||||
for _, c := range srcPodSpec.InitContainers {
|
||||
container, ok := targetContainers[c.Name]
|
||||
if ok {
|
||||
err := mergeContainer(&c, container)
|
||||
multierr = multierror.Append(multierr, err)
|
||||
delete(targetContainers, c.Name)
|
||||
}
|
||||
}
|
||||
|
||||
for _, container := range targetContainers {
|
||||
srcPodSpec.InitContainers = append(srcPodSpec.InitContainers, container)
|
||||
}
|
||||
return multierr.ErrorOrNil()
|
||||
}
|
||||
|
||||
func mergeVolumeLists(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
|
||||
volumeList := targetPodSpec.Volumes
|
||||
specVolumes := make(map[string]apiv1.Volume)
|
||||
for _, vol := range volumeList {
|
||||
specVolumes[vol.Name] = vol
|
||||
}
|
||||
|
||||
var multierr *multierror.Error
|
||||
for _, vol := range srcPodSpec.Volumes {
|
||||
_, ok := specVolumes[vol.Name]
|
||||
if ok {
|
||||
multierr = multierror.Append(multierr, errors.New("Duplicate volume name found in the spec"))
|
||||
} else {
|
||||
delete(specVolumes, vol.Name)
|
||||
}
|
||||
}
|
||||
|
||||
for _, volume := range specVolumes {
|
||||
srcPodSpec.Volumes = append(srcPodSpec.Volumes, volume)
|
||||
}
|
||||
return multierr.ErrorOrNil()
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
/*
|
||||
Copyright 2016 The Fission Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
package util
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func TestMergeContainerSpecs(t *testing.T) {
|
||||
expected := apiv1.Container{
|
||||
Name: "containerName",
|
||||
Image: "testImage",
|
||||
Command: []string{
|
||||
"command",
|
||||
},
|
||||
Args: []string{
|
||||
"arg1",
|
||||
"arg2",
|
||||
},
|
||||
ImagePullPolicy: apiv1.PullNever,
|
||||
TTY: true,
|
||||
Env: []apiv1.EnvVar{
|
||||
{
|
||||
Name: "a",
|
||||
Value: "b",
|
||||
},
|
||||
{
|
||||
Name: "c",
|
||||
Value: "d",
|
||||
},
|
||||
},
|
||||
}
|
||||
specs := []*apiv1.Container{
|
||||
{
|
||||
Name: "containerName",
|
||||
Image: "testImage",
|
||||
Command: []string{
|
||||
"command",
|
||||
},
|
||||
Args: []string{
|
||||
"arg1",
|
||||
"arg2",
|
||||
},
|
||||
ImagePullPolicy: apiv1.PullNever,
|
||||
TTY: true,
|
||||
},
|
||||
{
|
||||
Name: "shouldNotBeThere",
|
||||
Image: "shouldNotBeThere",
|
||||
Env: []apiv1.EnvVar{
|
||||
{
|
||||
Name: "a",
|
||||
Value: "b",
|
||||
},
|
||||
},
|
||||
ImagePullPolicy: apiv1.PullAlways,
|
||||
TTY: false,
|
||||
},
|
||||
{
|
||||
Env: []apiv1.EnvVar{
|
||||
{
|
||||
Name: "c",
|
||||
Value: "d",
|
||||
},
|
||||
},
|
||||
ImagePullPolicy: apiv1.PullIfNotPresent,
|
||||
TTY: false,
|
||||
},
|
||||
}
|
||||
result := MergeContainerSpecs(specs...)
|
||||
assert.Equal(t, expected, result)
|
||||
|
||||
// Check if merging order actually matters
|
||||
var rspecs []*apiv1.Container
|
||||
for i := len(specs) - 1; i >= 0; i -= 1 {
|
||||
rspecs = append(rspecs, specs[i])
|
||||
}
|
||||
reverseResult := MergeContainerSpecs(rspecs...)
|
||||
assert.NotEqual(t, expected, reverseResult)
|
||||
}
|
||||
|
||||
func TestMergeContainerSpecsSingle(t *testing.T) {
|
||||
expected := apiv1.Container{
|
||||
Name: "containerName",
|
||||
Image: "testImage",
|
||||
Command: []string{
|
||||
"command",
|
||||
},
|
||||
Args: []string{
|
||||
"arg1",
|
||||
"arg2",
|
||||
},
|
||||
ImagePullPolicy: apiv1.PullNever,
|
||||
TTY: true,
|
||||
}
|
||||
result := MergeContainerSpecs(&expected)
|
||||
assert.EqualValues(t, expected, result)
|
||||
}
|
||||
|
||||
func TestMergeContainerSpecsNil(t *testing.T) {
|
||||
expected := apiv1.Container{}
|
||||
result := MergeContainerSpecs()
|
||||
assert.EqualValues(t, expected, result)
|
||||
}
|
||||
Reference in New Issue
Block a user