Add kubernetes podSpec support in fission spec (#1106)

This commit is contained in:
Vishal
2019-05-29 03:31:14 +08:00
committed by Ta-Ching Chen
parent 49f9944b86
commit d578539391
19 changed files with 578 additions and 30 deletions
+2 -1
View File
@@ -35,6 +35,7 @@ import (
"github.com/fission/fission"
"github.com/fission/fission/crd"
fetcherConfig "github.com/fission/fission/environments/fetcher/config"
"github.com/fission/fission/executor/util"
)
type requestType int
@@ -515,7 +516,7 @@ func (envw *environmentWatcher) createBuilderDeployment(env *crd.Environment, ns
},
Spec: apiv1.PodSpec{
Containers: []apiv1.Container{
fission.MergeContainerSpecs(&apiv1.Container{
util.MergeContainerSpecs(&apiv1.Container{
Name: "builder",
Image: env.Spec.Builder.Image,
ImagePullPolicy: envw.builderImagePullPolicy,
+2 -23
View File
@@ -29,12 +29,10 @@ import (
"strings"
"syscall"
"go.uber.org/zap"
"github.com/gorilla/handlers"
"github.com/imdario/mergo"
"github.com/mholt/archiver"
"github.com/satori/go.uuid"
uuid "github.com/satori/go.uuid"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@@ -97,25 +95,6 @@ func LoggingMiddleware(logger *zap.Logger) func(next http.Handler) http.Handler
}
}
// MergeContainerSpecs merges container specs using a predefined order.
//
// The order of the arguments indicates which spec has precedence (lower index takes precedence over higher indexes).
// Slices and maps are merged; other fields are set only if they are a zero value.
func MergeContainerSpecs(specs ...*apiv1.Container) apiv1.Container {
result := &apiv1.Container{}
for _, spec := range specs {
if spec == nil {
continue
}
err := mergo.Merge(result, spec)
if err != nil {
panic(err)
}
}
return *result
}
// IsNetworkDialError returns true if its a network dial error
func IsNetworkDialError(err error) bool {
netErr, ok := err.(net.Error)
+14 -2
View File
@@ -32,6 +32,7 @@ import (
"github.com/fission/fission"
"github.com/fission/fission/crd"
"github.com/fission/fission/executor/util"
multierror "github.com/hashicorp/go-multierror"
)
@@ -200,7 +201,7 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen
},
Spec: apiv1.PodSpec{
Containers: []apiv1.Container{
fission.MergeContainerSpecs(&apiv1.Container{
util.MergeContainerSpecs(&apiv1.Container{
Name: fn.Metadata.Name,
Image: env.Spec.Runtime.Image,
ImagePullPolicy: deploy.runtimeImagePullPolicy,
@@ -225,12 +226,23 @@ func (deploy *NewDeploy) getDeploymentSpec(fn *crd.Function, env *crd.Environmen
},
}
deploy.fetcherConfig.AddSpecializingFetcherToPodSpec(
// Order of merging is important here - first fetcher, then containers and lastly pod spec
err := deploy.fetcherConfig.AddSpecializingFetcherToPodSpec(
&deployment.Spec.Template.Spec,
fn.Metadata.Name,
fn,
env,
)
if err != nil {
return nil, err
}
if env.Spec.Runtime.PodSpec != nil {
err := util.MergePodSpec(&deployment.Spec.Template.Spec, env.Spec.Runtime.PodSpec)
if err != nil {
return nil, err
}
}
return deployment, nil
}
+10 -1
View File
@@ -41,6 +41,7 @@ import (
fetcherClient "github.com/fission/fission/environments/fetcher/client"
fetcherConfig "github.com/fission/fission/environments/fetcher/config"
"github.com/fission/fission/executor/fscache"
"github.com/fission/fission/executor/util"
)
type (
@@ -371,7 +372,7 @@ func (gp *GenericPool) createPool() error {
},
Spec: apiv1.PodSpec{
Containers: []apiv1.Container{
fission.MergeContainerSpecs(&apiv1.Container{
util.MergeContainerSpecs(&apiv1.Container{
Name: gp.env.Metadata.Name,
Image: gp.env.Spec.Runtime.Image,
ImagePullPolicy: gp.runtimeImagePullPolicy,
@@ -405,11 +406,19 @@ func (gp *GenericPool) createPool() error {
},
}
// Order of merging is important here - first fetcher, then containers and lastly pod spec
err := gp.fetcherConfig.AddFetcherToPodSpec(&deployment.Spec.Template.Spec, gp.env.Metadata.Name)
if err != nil {
return err
}
if gp.env.Spec.Runtime.PodSpec != nil {
err = util.MergePodSpec(&deployment.Spec.Template.Spec, gp.env.Spec.Runtime.PodSpec)
if err != nil {
return err
}
}
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment)
if err != nil {
gp.logger.Error("error creating deployment in kubernetes", zap.Error(err), zap.String("deployment", deployment.Name))
+219
View File
@@ -0,0 +1,219 @@
/*
Copyright 2016 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"errors"
"github.com/hashicorp/go-multierror"
"github.com/imdario/mergo"
apiv1 "k8s.io/api/core/v1"
)
// MergeContainerSpecs merges container specs using a predefined order.
//
// The order of the arguments indicates which spec has precedence (lower index takes precedence over higher indexes).
// Slices and maps are merged; other fields are set only if they are a zero value.
func MergeContainerSpecs(specs ...*apiv1.Container) apiv1.Container {
result := &apiv1.Container{}
for _, spec := range specs {
if spec == nil {
continue
}
err := mergo.Merge(result, spec)
if err != nil {
panic(err)
}
}
return *result
}
// mergeContainer is a specialized implementation of MergeContainerSpecs
func mergeContainer(deployContainer *apiv1.Container, containerSpec apiv1.Container) error {
if &containerSpec == nil {
return nil
}
if deployContainer.Name == containerSpec.Name {
volMap := make(map[string]*apiv1.VolumeMount)
for _, vol := range deployContainer.VolumeMounts {
volMap[vol.Name] = &vol
}
for _, specVol := range containerSpec.VolumeMounts {
_, ok := volMap[specVol.Name]
if ok {
return errors.New("Duplicate volume name found in the spec")
} else {
deployContainer.VolumeMounts = append(deployContainer.VolumeMounts, specVol)
}
}
deployContainer.Env = append(deployContainer.Env, containerSpec.Env...)
}
return nil
}
func MergePodSpec(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
if &targetPodSpec == nil {
return nil
}
var multierr *multierror.Error
// Get item from spec, if they exist in deployment - merge, else append
// Same pattern for all lists (Mergo can not handle lists)
// At some point this is better done with generics/reflection?
err := mergeContainerLists(srcPodSpec, targetPodSpec)
if err != nil {
return err
}
err = mergeInitContainerList(srcPodSpec, targetPodSpec)
if err != nil {
return err
}
// For volumes - if duplicate exist, throw error
err = mergeVolumeLists(srcPodSpec, targetPodSpec)
if err != nil {
return err
}
if targetPodSpec.NodeName != "" {
srcPodSpec.NodeName = targetPodSpec.NodeName
}
if targetPodSpec.Subdomain != "" {
srcPodSpec.Subdomain = targetPodSpec.Subdomain
}
if targetPodSpec.SchedulerName != "" {
srcPodSpec.SchedulerName = targetPodSpec.SchedulerName
}
if targetPodSpec.PriorityClassName != "" {
srcPodSpec.PriorityClassName = targetPodSpec.PriorityClassName
}
if targetPodSpec.TerminationGracePeriodSeconds != nil {
srcPodSpec.TerminationGracePeriodSeconds = targetPodSpec.TerminationGracePeriodSeconds
}
//TODO - Security context should be merged instead of overriding.
if targetPodSpec.SecurityContext != nil {
srcPodSpec.SecurityContext = targetPodSpec.SecurityContext
}
//TODO - Affinity should be merged instead of overriding.
if targetPodSpec.Affinity != nil {
srcPodSpec.Affinity = targetPodSpec.Affinity
}
if targetPodSpec.Hostname != "" {
srcPodSpec.Hostname = targetPodSpec.Hostname
}
for _, obj := range targetPodSpec.ImagePullSecrets {
srcPodSpec.ImagePullSecrets = append(srcPodSpec.ImagePullSecrets, obj)
}
for _, obj := range targetPodSpec.Tolerations {
srcPodSpec.Tolerations = append(srcPodSpec.Tolerations, obj)
}
for _, obj := range targetPodSpec.HostAliases {
srcPodSpec.HostAliases = append(srcPodSpec.HostAliases, obj)
}
err = mergo.Merge(&srcPodSpec.NodeSelector, targetPodSpec.NodeSelector)
if err != nil {
multierr = multierror.Append(multierr, err)
}
return multierr.ErrorOrNil()
}
func mergeContainerLists(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
targetSpecContainers := targetPodSpec.Containers
targetContainers := make(map[string]apiv1.Container)
for _, c := range targetSpecContainers {
targetContainers[c.Name] = c
}
var multierr *multierror.Error
for _, c := range srcPodSpec.Containers {
container, ok := targetContainers[c.Name]
if ok {
err := mergeContainer(&c, container)
multierr = multierror.Append(multierr, err)
delete(targetContainers, c.Name)
}
}
for _, container := range targetContainers {
srcPodSpec.Containers = append(srcPodSpec.Containers, container)
}
return multierr.ErrorOrNil()
}
func mergeInitContainerList(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
targetSpecContainers := targetPodSpec.InitContainers
targetContainers := make(map[string]apiv1.Container)
for _, c := range targetSpecContainers {
targetContainers[c.Name] = c
}
var multierr *multierror.Error
for _, c := range srcPodSpec.InitContainers {
container, ok := targetContainers[c.Name]
if ok {
err := mergeContainer(&c, container)
multierr = multierror.Append(multierr, err)
delete(targetContainers, c.Name)
}
}
for _, container := range targetContainers {
srcPodSpec.InitContainers = append(srcPodSpec.InitContainers, container)
}
return multierr.ErrorOrNil()
}
func mergeVolumeLists(srcPodSpec *apiv1.PodSpec, targetPodSpec *apiv1.PodSpec) error {
volumeList := targetPodSpec.Volumes
specVolumes := make(map[string]apiv1.Volume)
for _, vol := range volumeList {
specVolumes[vol.Name] = vol
}
var multierr *multierror.Error
for _, vol := range srcPodSpec.Volumes {
_, ok := specVolumes[vol.Name]
if ok {
multierr = multierror.Append(multierr, errors.New("Duplicate volume name found in the spec"))
} else {
delete(specVolumes, vol.Name)
}
}
for _, volume := range specVolumes {
srcPodSpec.Volumes = append(srcPodSpec.Volumes, volume)
}
return multierr.ErrorOrNil()
}
@@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package fission
package util
import (
"testing"
+4 -1
View File
@@ -32,7 +32,7 @@ import (
multierror "github.com/hashicorp/go-multierror"
"github.com/mholt/archiver"
"github.com/pkg/errors"
"github.com/satori/go.uuid"
uuid "github.com/satori/go.uuid"
"github.com/urfave/cli"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -395,6 +395,9 @@ func (fr *FissionResources) validate() error {
environments := make(map[string]struct{})
for _, e := range fr.environments {
environments[fmt.Sprintf("%s:%s", e.Metadata.Name, e.Metadata.Namespace)] = struct{}{}
if (e.Spec.Runtime.Container != nil) && (e.Spec.Runtime.PodSpec != nil) {
log.Warn("You have provided both - container spec and pod spec and while merging the pod spec will take precedence.")
}
}
for _, f := range fr.functions {
+2
View File
@@ -145,6 +145,7 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.8.0 h1:1921Yw9Gc3iSc4VQh3PIoOqgPCZS7G/4xQNVUp8Mda8=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -173,6 +174,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx
github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/ulikunitz/xz v0.0.0-20180703112113-636d36a76670 h1:HQWT4ta3wW5GZ790GaqLCS+w1dvuA3rMfEQxLi+UOYU=
github.com/ulikunitz/xz v0.0.0-20180703112113-636d36a76670/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
+10
View File
@@ -223,6 +223,16 @@ type (
// - ImagePullPolicy
// (optional)
Container *apiv1.Container `json:"container,omitempty"`
// Podspec allows modification of deployed runtime pod with Kubernetes PodSpec
// The merging logic is briefly described below and detailed MergePodSpec function
// - Volumes mounts and env variables for function and fetcher container are appended
// - All additional containers and init containers are appended
// - Volume definitions are appended
// - Lists such as tolerations, ImagePullSecrets, HostAliases are appended
// - Structs are merged and variables from pod spec take precedence
// (optional)
PodSpec *apiv1.PodSpec `json:"podspec,omitempty"`
}
Builder struct {
Binary file not shown.
+2 -1
View File
@@ -477,7 +477,8 @@ run_all_tests() {
$ROOT/test/tests/test_pass.sh \
$ROOT/test/tests/test_router_cache_invalidation.sh \
$ROOT/test/tests/test_specs/test_spec.sh \
$ROOT/test/tests/test_specs/test_spec_multifile.sh
$ROOT/test/tests/test_specs/test_spec_multifile.sh \
$ROOT/test/tests/test_specs/test_spec_merge/test_spec_merge.sh
FAILURES=$?
# FIXME: run tests with newdeploy one by one.
@@ -0,0 +1,7 @@
module.exports = async function(context) {
return {
status: 200,
body: "hello, world!\n"
};
}
@@ -0,0 +1,42 @@
Fission Specs
=============
This is a set of specifications for a Fission app. This includes functions,
environments, and triggers; we collectively call these things "resources".
How to use these specs
----------------------
These specs are handled with the 'fission spec' command. See 'fission spec --help'.
'fission spec apply' will "apply" all resources specified in this directory to your
cluster. That means it checks what resources exist on your cluster, what resources are
specified in the specs directory, and reconciles the difference by creating, updating or
deleting resources on the cluster.
'fission spec apply' will also package up your source code (or compiled binaries) and
upload the archives to the cluster if needed. It uses 'ArchiveUploadSpec' resources in
this directory to figure out which files to archive.
You can use 'fission spec apply --watch' to watch for file changes and continuously keep
the cluster updated.
You can add YAMLs to this directory by writing them manually, but it's easier to generate
them. Use 'fission function create --spec' to generate a function spec,
'fission environment create --spec' to generate an environment spec, and so on.
You can edit any of the files in this directory, except 'fission-deployment-config.yaml',
which contains a UID that you should never change. To apply your changes simply use
'fission spec apply'.
fission-deployment-config.yaml
------------------------------
fission-deployment-config.yaml contains a UID. This UID is what fission uses to correlate
resources on the cluster to resources in this directory.
All resources created by 'fission spec apply' are annotated with this UID. Resources on
the cluster that are _not_ annotated with this UID are never modified or deleted by
fission.
@@ -0,0 +1,72 @@
apiVersion: fission.io/v1
kind: Environment
metadata:
creationTimestamp: null
name: nodend
namespace: default
spec:
TerminationGracePeriod: 360
builder: {}
keeparchive: false
poolsize: 0
resources: {}
runtime:
functionendpointport: 0
image: fission/node-env
loadendpointpath: ""
loadendpointport: 0
container:
name: nodehellond
volumeMounts:
- name: cvol
mountPath: /etc/cvoldata
readOnly: true
podspec:
hostname: foo-bar
# A container which will be merged with for pool manager
Containers:
- name: nodehellond
image: fission/node-env
volumeMounts:
- name: funcvol
mountPath: /etc/funcdata
readOnly: true
# A additional container in the pods
- name: yanode
image: fission/node-env
command: ['sh', '-c', 'sleep 36000000000']
# Init container before function
initContainers:
- name: init-node
image: fission/node-env
command: ['sh', '-c', 'cat /etc/infopod/labels']
volumeMounts:
- name: infopod
mountPath: /etc/infopod
readOnly: false
tolerations:
- key: "reservation"
operator: "Equal"
value: "fission"
effect: "NoSchedule"
# Additional volumes
volumes:
- name: infopod
downwardAPI:
items:
- path: "labels"
fieldRef:
fieldPath: metadata.labels
- name: funcvol
downwardAPI:
items:
- path: "labels"
fieldRef:
fieldPath: metadata.labels
- name: cvol
downwardAPI:
items:
- path: "labels"
fieldRef:
fieldPath: metadata.labels
version: 1
@@ -0,0 +1,72 @@
apiVersion: fission.io/v1
kind: Environment
metadata:
creationTimestamp: null
name: nodep
namespace: default
spec:
TerminationGracePeriod: 360
builder: {}
keeparchive: false
poolsize: 3
resources: {}
runtime:
functionendpointport: 0
image: fission/node-env
loadendpointpath: ""
loadendpointport: 0
container:
name: nodep
volumeMounts:
- name: cvol
mountPath: /etc/cvoldata
readOnly: true
podspec:
hostname: foo-bar
# A container which will be merged with for pool manager
Containers:
- name: nodep
image: fission/node-env
volumeMounts:
- name: funcvol
mountPath: /etc/funcdata
readOnly: true
# A additional container in the pods
- name: yanode
image: fission/node-env
command: ['sh', '-c', 'sleep 36000000000']
# Init container before function
initContainers:
- name: init-node
image: fission/node-env
command: ['sh', '-c', 'cat /etc/infopod/labels']
volumeMounts:
- name: infopod
mountPath: /etc/infopod
readOnly: false
tolerations:
- key: "reservation"
operator: "Equal"
value: "fission"
effect: "NoSchedule"
# Additional volumes
volumes:
- name: infopod
downwardAPI:
items:
- path: "labels"
fieldRef:
fieldPath: metadata.labels
- name: funcvol
downwardAPI:
items:
- path: "labels"
fieldRef:
fieldPath: metadata.labels
- name: cvol
downwardAPI:
items:
- path: "labels"
fieldRef:
fieldPath: metadata.labels
version: 1
@@ -0,0 +1,7 @@
# This file is generated by the 'fission spec init' command.
# See the README in this directory for background and usage information.
# Do not edit the UID below: that will break 'fission spec apply'
apiVersion: fission.io/v1
kind: DeploymentConfig
name: spec-merge
uid: b1573a35-f3a2-45a8-a430-1f1a08d71177
@@ -0,0 +1,24 @@
apiVersion: fission.io/v1
kind: Function
metadata:
creationTimestamp: null
name: nodehellond
namespace: default
spec:
InvokeStrategy:
ExecutionStrategy:
ExecutorType: newdeploy
MaxScale: 3
MinScale: 1
TargetCPUPercent: 80
StrategyType: execution
configmaps: null
environment:
name: nodend
namespace: default
package:
packageref:
name: hello-js-vm2y
namespace: default
resources: {}
secrets: null
@@ -0,0 +1,50 @@
include:
- hello.js
kind: ArchiveUploadSpec
name: hello-js-leSC
---
apiVersion: fission.io/v1
kind: Package
metadata:
creationTimestamp: null
name: hello-js-vm2y
namespace: default
spec:
deployment:
checksum: {}
type: url
url: archive://hello-js-leSC
environment:
name: nodep
namespace: default
source:
checksum: {}
status:
buildstatus: none
---
apiVersion: fission.io/v1
kind: Function
metadata:
creationTimestamp: null
name: nodehellop
namespace: default
spec:
InvokeStrategy:
ExecutionStrategy:
ExecutorType: poolmgr
MaxScale: 0
MinScale: 0
TargetCPUPercent: 0
StrategyType: execution
configmaps: null
environment:
name: nodep
namespace: default
package:
packageref:
name: hello-js-vm2y
namespace: default
resources: {}
secrets: null
+38
View File
@@ -0,0 +1,38 @@
#!/bin/bash
set -euo pipefail
env_p=nodep
fn_p=nodehellop
env_n=nodend
fn_nd=nodehellond
cleanup() {
echo "Cleaning up..."
popd
fission spec destroy || true
}
trap cleanup EXIT
pushd $(dirname $0)
fission spec apply
fission fn test --name $fn_p
fission fn test --name $fn_nd
hnd=$(kubectl -n $FUNCTION_NAMESPACE get deployment -l=functionName=$fn_nd -ojsonpath='{.items[0].spec.template.spec.hostname}')
if [[ "${hnd}" == "foo-bar" ]]
then
echo "Hostname matches for newdeployment function, podspec test 1/2 passsed"
fi
hnp=$(kubectl -n $FUNCTION_NAMESPACE get deployment -l=environmentName=$env_p -ojsonpath='{.items[0].spec.template.spec.hostname}')
if [[ "${hnp}" == "foo-bar" ]]
then
echo "Hostname matches for poolmgr function, podspec test 2/2 passsed"
fi