Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #491 from fission/ciImprovements

Browse Source 
CI modifications
This commit is contained in:
smruthi2187
2018-02-09 19:00:19 -08:00
committed by GitHub
parent 4d0e6af5de 281a492149
commit 9e9a0c1ec6
26 changed files with 442 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+5
View File
@@ -2,6 +2,10 @@ sudo: required
dist: trusty
branches:
only:
- master
language: go
go:
@@ -18,6 +22,7 @@ services:
before_install:
- sudo apt-get update
- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
- sudo sysctl net.ipv6.conf.all.disable_ipv6=0
install:
- go get github.com/Masterminds/glide
charts/fission-all/templates/deployment.yaml
+52
View File
@@ -129,6 +129,19 @@ spec:
imagePullPolicy: {{ .Values.pullPolicy }}
command: ["/fission-bundle"]
args: ["--controllerPort", "8888"]
readinessProbe:
httpGet:
path: "/healthz"
port: 8888
initialDelaySeconds: 1
periodSeconds: 1
failureThreshold: 30
livenessProbe:
httpGet:
path: "/healthz"
port: 8888
initialDelaySeconds: 35
periodSeconds: 5
serviceAccount: fission-svc
---
@@ -151,6 +164,19 @@ spec:
imagePullPolicy: {{ .Values.pullPolicy }}
command: ["/fission-bundle"]
args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"]
readinessProbe:
httpGet:
path: "/router-healthz"
port: 8888
initialDelaySeconds: 1
periodSeconds: 1
failureThreshold: 30
livenessProbe:
httpGet:
path: "/router-healthz"
port: 8888
initialDelaySeconds: 35
periodSeconds: 5
serviceAccount: fission-svc
---
@@ -196,6 +222,19 @@ spec:
value: "{{ .Values.pullPolicy }}"
- name: RUNTIME_IMAGE_PULL_POLICY
value: "{{ .Values.pullPolicy }}"
readinessProbe:
httpGet:
path: "/healthz"
port: 8888
initialDelaySeconds: 1
periodSeconds: 1
failureThreshold: 30
livenessProbe:
httpGet:
path: "/healthz"
port: 8888
initialDelaySeconds: 35
periodSeconds: 5
serviceAccount: fission-svc
---
@@ -535,6 +574,19 @@ spec:
volumeMounts:
- name: fission-storage
mountPath: /fission
readinessProbe:
httpGet:
path: "/healthz"
port: 8000
initialDelaySeconds: 1
periodSeconds: 1
failureThreshold: 30
livenessProbe:
httpGet:
path: "/healthz"
port: 8000
initialDelaySeconds: 35
periodSeconds: 5
serviceAccount: fission-svc
volumes:
- name: fission-storage
charts/fission-core/templates/deployment.yaml
+39
View File
@@ -129,6 +129,19 @@ spec:
imagePullPolicy: {{ .Values.pullPolicy }}
command: ["/fission-bundle"]
args: ["--controllerPort", "8888"]
readinessProbe:
httpGet:
path: "/healthz"
port: "8888"
initialDelaySeconds: 1
periodSeconds: 1
failureThreshold: 30
livenessProbe:
httpGet:
path: "/healthz"
port: "8888"
initialDelaySeconds: 35
periodSeconds: 5
serviceAccount: fission-svc
---
@@ -151,6 +164,19 @@ spec:
imagePullPolicy: {{ .Values.pullPolicy }}
command: ["/fission-bundle"]
args: ["--routerPort", "8888", "--executorUrl", "http://executor.{{ .Release.Namespace }}"]
readinessProbe:
httpGet:
path: "/router-healthz"
port: "8888"
initialDelaySeconds: 1
periodSeconds: 1
failureThreshold: 30
livenessProbe:
httpGet:
path: "/router-healthz"
port: "8888"
initialDelaySeconds: 35
periodSeconds: 5
serviceAccount: fission-svc
---
@@ -194,6 +220,19 @@ spec:
value: "{{ .Values.fetcherImage }}:{{ .Values.fetcherImageTag }}"
- name: FETCHER_IMAGE_PULL_POLICY
value: "{{ .Values.pullPolicy }}"
readinessProbe:
httpGet:
path: "/healthz"
port: "8888"
initialDelaySeconds: 1
periodSeconds: 1
failureThreshold: 30
livenessProbe:
httpGet:
path: "/healthz"
port: "8888"
initialDelaySeconds: 35
periodSeconds: 5
serviceAccount: fission-svc
---
common.go
+16
View File
@@ -18,9 +18,25 @@ package fission
import (
"fmt"
"os"
"os/signal"
"runtime/debug"
"syscall"
)
func UrlForFunction(name string) string {
prefix := "/fission-function"
return fmt.Sprintf("%v/%v", prefix, name)
}
func SetupStackTraceHandler() {
// register signal handler for dumping stack trace.
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGTERM)
go func() {
<-c
fmt.Println("Received SIGTERM : Dumping stack trace")
debug.PrintStack()
os.Exit(1)
}()
}
controller/api.go
+5
View File
@@ -128,8 +128,13 @@ func (api *API) ApiVersionMismatchHandler(w http.ResponseWriter, r *http.Request
api.respondWithError(w, err)
}
func (api *API) HealthHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func (api *API) Serve(port int) {
r := mux.NewRouter()
r.HandleFunc("/healthz", api.HealthHandler).Methods("GET")
// Give a useful error message if an older CLI attempts to make a request
r.HandleFunc(`/v1/{rest:[a-zA-Z0-9=\-\/]+}`, api.ApiVersionMismatchHandler)
r.HandleFunc("/", api.HomeHandler)
controller/controller.go
+4
View File
@@ -19,10 +19,14 @@ package controller
import (
"log"
"github.com/fission/fission"
"github.com/fission/fission/crd"
)
func Start(port int) {
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
fc, _, apiExtClient, err := crd.MakeFissionClient()
if err != nil {
log.Fatalf("Failed to connect to K8s API: %v", err)
environments/fetcher/cmd/main.go
+23 -1
View File
@@ -10,15 +10,32 @@ import (
"net/http"
"net/url"
"os"
"os/signal"
"runtime/debug"
"strconv"
"syscall"
"time"
"github.com/fission/fission"
"github.com/fission/fission/environments/fetcher"
)
func dumpStackTrace() {
debug.PrintStack()
}
// Usage: fetcher <shared volume path>
func main() {
// register signal handler for dumping stack trace.
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGTERM)
go func() {
<-c
log.Println("Received SIGTERM : Dumping stack trace")
dumpStackTrace()
os.Exit(1)
}()
flag.Usage = fetcherUsage
specializeOnStart := flag.Bool("specialize-on-startup", false, "Flag to activate specialize process at pod starup")
fetchPayload := flag.String("fetch-request", "", "JSON Payload for fetch request")
@@ -42,7 +59,10 @@ func main() {
}
}
fetcher := fetcher.MakeFetcher(dir, *secretDir, *configDir)
fetcher, err := fetcher.MakeFetcher(dir, *secretDir, *configDir)
if err != nil {
log.Fatalf("Error making fetcher: %v", err)
}
if *specializeOnStart {
specializePod(fetcher, fetchPayload, loadPayload)
@@ -54,6 +74,8 @@ func main() {
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
})
log.Println("Fetcher ready to receive requests")
http.ListenAndServe(":8000", mux)
}
environments/fetcher/fetcher.go
+3 -3
View File
@@ -74,14 +74,14 @@ func makeVolumeDir(dirPath string) {
}
}
func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) *Fetcher {
func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigPath string) (*Fetcher, error) {
makeVolumeDir(sharedVolumePath)
makeVolumeDir(sharedSecretPath)
makeVolumeDir(sharedConfigPath)
fissionClient, kubeClient, _, err := crd.MakeFissionClient()
if err != nil {
return nil
return nil, err
}
return &Fetcher{
sharedVolumePath: sharedVolumePath,
@@ -89,7 +89,7 @@ func MakeFetcher(sharedVolumePath string, sharedSecretPath string, sharedConfigP
sharedConfigPath: sharedConfigPath,
fissionClient: fissionClient,
kubeClient: kubeClient,
}
}, nil
}
func downloadUrl(url string, localPath string) error {
executor/api.go
+5
View File
@@ -99,10 +99,15 @@ func (executor *Executor) tapService(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func (executor *Executor) healthHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func (executor *Executor) Serve(port int) {
r := mux.NewRouter()
r.HandleFunc("/v2/getServiceForFunction", executor.getServiceForFunctionApi).Methods("POST")
r.HandleFunc("/v2/tapService", executor.tapService).Methods("POST")
r.HandleFunc("/healthz", executor.healthHandler).Methods("GET")
address := fmt.Sprintf(":%v", port)
log.Printf("starting executor at port %v", port)
ctx, cancel := context.WithCancel(context.Background())
executor/executor.go
+8
View File
@@ -18,6 +18,7 @@ package executor
import (
"log"
"runtime/debug"
"strings"
"sync"
"time"
@@ -181,9 +182,16 @@ func (executor *Executor) getFunctionEnv(m *metav1.ObjectMeta) (*crd.Environment
return env, nil
}
func dumpStackTrace() {
debug.PrintStack()
}
// StartExecutor Starts executor and the executor components such as Poolmgr,
// deploymgr and potential future executor types
func StartExecutor(fissionNamespace string, functionNamespace string, port int) error {
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
fissionClient, kubernetesClient, _, err := crd.MakeFissionClient()
restClient := fissionClient.GetCrdClient()
if err != nil {
executor/poolmgr/gp.go
+28
View File
@@ -519,6 +519,33 @@ func (gp *GenericPool) createPool() error {
"-secret-dir", gp.sharedSecretPath,
"-cfgmap-dir", gp.sharedCfgMapPath,
gp.sharedMountPath},
ReadinessProbe: &apiv1.Probe{
InitialDelaySeconds: 1,
PeriodSeconds: 1,
FailureThreshold: 30,
Handler: apiv1.Handler{
HTTPGet: &apiv1.HTTPGetAction{
Path: "/healthz",
Port: intstr.IntOrString{
Type: intstr.Int,
IntVal: 8000,
},
},
},
},
LivenessProbe: &apiv1.Probe{
InitialDelaySeconds: 35,
PeriodSeconds: 5,
Handler: apiv1.Handler{
HTTPGet: &apiv1.HTTPGetAction{
Path: "/healthz",
Port: intstr.IntOrString{
Type: intstr.Int,
IntVal: 8000,
},
},
},
},
},
},
ServiceAccountName: "fission-fetcher",
@@ -528,6 +555,7 @@ func (gp *GenericPool) createPool() error {
}
depl, err := gp.kubernetesClient.ExtensionsV1beta1().Deployments(gp.namespace).Create(deployment)
if err != nil {
log.Printf("Error creating deployment for %s in kubernetes, err: %v", deployment.Name, err)
return err
}
gp.deployment = depl
router/httpTriggers.go
+7
View File
@@ -89,6 +89,10 @@ func defaultHomeHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func routerHealthHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func (ts *HTTPTriggerSet) getRouter() *mux.Router {
muxRouter := mux.NewRouter()
@@ -150,6 +154,9 @@ func (ts *HTTPTriggerSet) getRouter() *mux.Router {
muxRouter.HandleFunc(fission.UrlForFunction(function.Metadata.Name), fh.handler)
}
// Healthz endpoint for the router.
muxRouter.HandleFunc("/router-healthz", routerHealthHandler).Methods("GET")
return muxRouter
}
router/router.go
+4
View File
@@ -50,6 +50,7 @@ import (
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/fission/fission"
"github.com/fission/fission/crd"
executorClient "github.com/fission/fission/executor/client"
)
@@ -72,6 +73,9 @@ func serve(ctx context.Context, port int, httpTriggerSet *HTTPTriggerSet, resolv
}
func Start(port int, executorUrl string) {
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
fmap := makeFunctionServiceMap(time.Minute)
fissionClient, _, _, err := crd.MakeFissionClient()
storagesvc/storagesvc.go
+9
View File
@@ -25,6 +25,7 @@ import (
"strconv"
"time"
"github.com/fission/fission"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
_ "github.com/graymeta/stow/local"
@@ -149,6 +150,10 @@ func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request
}
}
func (ss *StorageService) healthHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
func MakeStorageService(storageClient *StowClient, port int) *StorageService {
return &StorageService{
storageClient: storageClient,
@@ -161,12 +166,16 @@ func (ss *StorageService) Start(port int) {
r.HandleFunc("/v1/archive", ss.uploadHandler).Methods("POST")
r.HandleFunc("/v1/archive", ss.downloadHandler).Methods("GET")
r.HandleFunc("/v1/archive", ss.deleteHandler).Methods("DELETE")
r.HandleFunc("/healthz", ss.healthHandler).Methods("GET")
address := fmt.Sprintf(":%v", port)
log.Fatal(http.ListenAndServe(address, handlers.LoggingHandler(os.Stdout, r)))
}
func RunStorageService(storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService {
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
// initialize logger
log.SetLevel(log.InfoLevel)
test/test_utils.sh
+94 -25
View File
@@ -59,7 +59,7 @@ build_and_push_fission_bundle() {
pushd $ROOT/fission-bundle
./build.sh
docker build -t $image_tag .
docker build -q -t $image_tag .
gcloud_login
@@ -72,7 +72,7 @@ build_and_push_fetcher() {
pushd $ROOT/environments/fetcher/cmd
./build.sh
docker build -t $image_tag .
docker build -q -t $image_tag .
gcloud_login
@@ -86,7 +86,7 @@ build_and_push_builder() {
pushd $ROOT/builder/cmd
./build.sh
docker build -t $image_tag .
docker build -q -t $image_tag .
gcloud_login
@@ -98,7 +98,7 @@ build_and_push_fluentd(){
image_tag=$1
pushd $ROOT/logger/fluentd
docker build -t $image_tag .
docker build -q -t $image_tag .
gcloud_login
@@ -112,7 +112,7 @@ build_and_push_env_runtime() {
image_tag=$2
pushd $ROOT/environments/$env/
docker build -t $image_tag .
docker build -q -t $image_tag .
gcloud_login
@@ -127,7 +127,7 @@ build_and_push_env_builder() {
pushd $ROOT/environments/$env/builder
docker build -t $image_tag --build-arg BUILDER_IMAGE=${builder_image} .
docker build -q -t $image_tag --build-arg BUILDER_IMAGE=${builder_image} .
gcloud_login
@@ -174,14 +174,19 @@ helm_install_fission() {
echo "Deleting old releases"
helm list -q|xargs -I@ bash -c "helm_uninstall_fission @"
# deleting ns does take a while after command is issued
while `kubectl get ns| grep fission-builder`
do
sleep 5
done
echo "Installing fission"
helm install \
--wait \
--timeout 600 \
--timeout 540 \
--name $id \
--set $helmVars \
--namespace $ns \
--debug \
$ROOT/charts/fission-all
helm list
@@ -190,30 +195,65 @@ helm_install_fission() {
wait_for_service() {
id=$1
svc=$2
health_endpoint=$3
ns=f-$id
retry=0
max_retries=5
while true
do
ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}')
if [ ! -z $ip ]
then
break
fi
echo Waiting for service $svc...
sleep 1
retry=$((retry+1))
if ((retry == max_retries)); then
echo "Waiting for $svc to be routable exceeded max retries. Quitting.."
exit 1
fi
ip=$(kubectl -n $ns get svc $svc -o jsonpath='{...ip}')
if [ -z $ip ]; then
continue
fi
http_status=`curl -sw "%{http_code}" "http://$ip/$health_endpoint"`
echo "http_status for svc $svc : $http_status"
if [ "$http_status" -ne "200" ]; then
echo "Service $svc returned response other than 200. waiting for 200 after backing off for 1 second"
sleep 1
else
break
fi
done
}
wait_for_services() {
id=$1
wait_for_service $id controller
wait_for_service $id router
echo Waiting for service is routable...
sleep 10
echo "\n--- wait for controller and router services to be routable ---"
wait_for_service $id controller "healthz"
wait_for_service $id router "router-healthz"
echo "\n--- end wait for controller and router services to be routable ---"
}
dump_kubernetes_events() {
id=$1
ns=f-$id
fns=f-func-$id
echo "--- kubectl events $fns ---"
kubectl get events -n $fns
echo "--- end kubectl events $fns ---"
echo "--- kubectl events $ns ---"
kubectl get events -n $ns
echo "--- end kubectl events $ns ---"
}
export -f dump_kubernetes_events
dump_tiller_logs() {
echo "--- tiller logs ---"
tiller_pod=`kubectl get pods -n kube-system | grep tiller| tr -s " "| cut -d" " -f1`
kubectl logs $tiller_pod --since=30m -n kube-system
echo "--- end tiller logs ---"
}
export -f dump_tiller_logs
helm_uninstall_fission() {(set +e
id=$1
@@ -222,10 +262,10 @@ helm_uninstall_fission() {(set +e
echo "Fission uninstallation skipped"
return
fi
echo "Uninstalling fission"
helm delete --purge $id
kubectl delete ns f-$id
kubectl delete ns f-$id || true
)}
export -f helm_uninstall_fission
@@ -243,7 +283,7 @@ set_environment() {
dump_builder_pod_logs() {
bns=$1
builderPods=$(kubectl -n $bns get pod -o name)
for p in $builderPods
do
echo "--- builder pod logs $p ---"
@@ -313,11 +353,30 @@ dump_env_pods() {
echo --- End environment pods ---
}
describe_pods_ns() {
echo "--- describe pods $1---"
kubectl describe pods -n $1
echo "--- End describe pods $1 ---"
}
describe_all_pods() {
id=$1
ns=f-$id
fns=f-func-$id
bns=fission-builder
describe_pods_ns $ns
describe_pods_ns $fns
describe_pods_ns $bns
}
dump_all_fission_resources() {
ns=$1
echo "--- All objects in the fission namespace $ns ---"
kubectl -n $ns get all
kubectl -n $ns get pods -o wide
echo ""
kubectl -n $ns get svc
echo "--- End objects in the fission namespace $ns ---"
}
@@ -343,11 +402,17 @@ dump_logs() {
dump_fission_logs $ns $fns router
dump_fission_logs $ns $fns buildermgr
dump_fission_logs $ns $fns executor
dump_fission_logs $ns $fns storagesvc
dump_function_pod_logs $ns $fns
dump_builder_pod_logs $bns
dump_fission_crds
}
log() {
echo `date +%Y/%m/%d:%H:%M:%S`" $1"
}
export -f log
export FAILURES=0
run_all_tests() {
@@ -402,7 +467,9 @@ install_and_test() {
trap "helm_uninstall_fission $id" EXIT
if ! helm_install_fission $id $image $imageTag $fetcherImage $fetcherImageTag $controllerPort $routerPort $fluentdImage $fluentdImageTag $pruneInterval
then
dump_logs $id
describe_all_pods $id
dump_kubernetes_events $id
dump_tiller_logs
exit 1
fi
@@ -417,6 +484,8 @@ install_and_test() {
if [ $FAILURES -ne 0 ]
then
# describe each pod in fission ns and function namespace
describe_all_pods $id
exit 1
fi
}
test/tests/test_archive_pruner.sh
+21 -21
View File
@@ -16,7 +16,7 @@ cleanup() {
}
create_archive() {
echo "Creating an archive"
log "Creating an archive"
mkdir test_dir
dd if=/dev/urandom of=test_dir/dynamically_generated_file bs=256k count=1
printf 'def main():\n return "Hello, world!"' > test_dir/hello.py
@@ -24,17 +24,17 @@ create_archive() {
}
create_package() {
echo "Creating package"
log "Creating package"
pkg=$(fission package create --deploy "test-deploy-pkg.zip" --env python| cut -f2 -d' '| tr -d \')
}
delete_package() {
echo "Deleting package: $1"
log "Deleting package: $1"
fission package delete --name $1
}
get_archive_url_from_package() {
echo "Getting archive URL from package: $1"
log "Getting archive URL from package: $1"
url=`kubectl get package $1 -ojsonpath='{.spec.deployment.url}'`
}
@@ -42,76 +42,76 @@ get_archive_from_storage() {
http_status=`curl -sw "%{http_code}" $1 -o /tmp/file`
}
#1. declare trap to cleanup for all the required signals
#2. create an archives with large files such that total size of archive is > 256KB
#1. declare trap to cleanup for EXIT
#2. create an archive with large files such that total size of archive is > 256KB
#3. create 2 pkgs referencing those archives
#4. delete both the packages
#5. verify archives are not recycled . this handles the case where archives are just created but not referenced by pkgs yet.
#6. sleep for two minutes
#7. now verify that both get deleted.
#7. now verify that both got deleted.
main() {
# trap
trap cleanup EXIT
# create a huge archive
create_archive
echo "created archive test-deploy-pkg.zip"
log "created archive test-deploy-pkg.zip"
# create packages with the huge archive
create_package
pkg_1=$pkg
get_archive_url_from_package $pkg_1
url_1=$url
echo "pkg: $pkg_1, archive_url : $url_1"
log "pkg: $pkg_1, archive_url : $url_1"
create_package
pkg_2=$pkg
get_archive_url_from_package $pkg_2
url_2=$url
echo "pkg: $pkg_2, archive_url : $url_2"
log "pkg: $pkg_2, archive_url : $url_2"
# delete packages
delete_package $pkg_1
delete_package $pkg_2
echo "deleted packages : $pkg_1 $pkg_2"
log "deleted packages : $pkg_1 $pkg_2"
# curl on the archive url
get_archive_from_storage $url_1
echo "http_status for $url_1 : $http_status"
log "http_status for $url_1 : $http_status"
if [ "$http_status" -ne "200" ]; then
echo "Archive $url_1 absent on storage, while expected to be present"
log "Archive $url_1 absent on storage, while expected to be present"
exit 1
fi
# curl on the archive url
get_archive_from_storage $url_2
echo "http_status for $url_2 : $http_status"
log "http_status for $url_2 : $http_status"
if [ "$http_status" -ne "200" ]; then
echo "Archive $url_2 absent on storage, while expected to be present"
log "Archive $url_2 absent on storage, while expected to be present"
exit 1
fi
# archivePruner is set to run every minute for test. In production, its set to run every hour.
echo "waiting for packages to get recycled"
log "waiting for packages to get recycled"
sleep 120
# curl on the archive url
get_archive_from_storage $url_1
echo "http_status for $url_1 : $http_status"
log "http_status for $url_1 : $http_status"
if [ "$http_status" -ne "404" ]; then
echo "Archive $url_1 should have been recycled, but curl returned $http_status, while expected status is 404."
log "Archive $url_1 should have been recycled, but curl returned $http_status, while expected status is 404."
exit 1
fi
# curl on the archive url
get_archive_from_storage $url_2
echo "http_status for $url_2 : $http_status"
log "http_status for $url_2 : $http_status"
if [ "$http_status" -ne "404" ]; then
echo "Archive $url_2 should have been recycled, but curl returned $http_status, while expected status is 404."
log "Archive $url_2 should have been recycled, but curl returned $http_status, while expected status is 404."
exit 1
fi
echo "Test archive pruner PASSED"
log "Test archive pruner PASSED"
}
main
test/tests/test_backend_newdeploy.sh
+13 -13
View File
@@ -5,51 +5,51 @@ set -euo pipefail
ROOT=$(dirname $0)/../..
# Create a hello world function in nodejs, test it with an http trigger
echo "NewDeploy ExecutorType: Pre-test cleanup"
log "NewDeploy ExecutorType: Pre-test cleanup"
fission env delete --name nodejs || true
echo "Creating nodejs env"
log "Creating nodejs env"
fission env create --name nodejs --image fission/node-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
trap "fission env delete --name nodejs" EXIT
# TODO Imporve test code by reusing common blocks
echo "Creating function, testing for cold start with MinScale 0"
log "Creating function, testing for cold start with MinScale 0"
fn0=nodejs-hello-$(date +%N)
fission fn create --name $fn0 --env nodejs --code $ROOT/examples/nodejs/hello.js --minscale 0 --maxscale 4 --executortype newdeploy
trap "fission fn delete --name $fn0" EXIT
echo "Creating route"
log "Creating route"
fission route create --function $fn0 --url /$fn0 --method GET
echo "Waiting for router & newdeploy deployment creation"
log "Waiting for router & newdeploy deployment creation"
sleep 5
echo "Doing an HTTP GET on the function's route"
log "Doing an HTTP GET on the function's route"
response0=$(curl http://$FISSION_ROUTER/$fn0)
echo "Checking for valid response"
log "Checking for valid response"
echo $response0 | grep -i hello
echo "Creating function, testing for warm start with MinScale 1"
log "Creating function, testing for warm start with MinScale 1"
fn1=nodejs-hello-$(date +%N)
fission fn create --name $fn1 --env nodejs --code $ROOT/examples/nodejs/hello.js --minscale 1 --maxscale 4 --executortype newdeploy
trap "fission fn delete --name $fn1" EXIT
echo "Creating route"
log "Creating route"
fission route create --function $fn1 --url /$fn1 --method GET
echo "Waiting for router & newdeploy deployment creation"
log "Waiting for router & newdeploy deployment creation"
sleep 5
echo "Doing an HTTP GET on the function's route"
log "Doing an HTTP GET on the function's route"
response1=$(curl http://$FISSION_ROUTER/$fn0)
echo "Checking for valid response"
log "Checking for valid response"
echo $response1 | grep -i hello
# crappy cleanup, improve this later
kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
echo "NewDeploy ExecutorType: All done."
log "NewDeploy ExecutorType: All done."
test/tests/test_backend_poolmgr.sh
+8 -8
View File
@@ -7,30 +7,30 @@ ROOT=$(dirname $0)/../..
fn=nodejs-hello-$(date +%N)
# Create a hello world function in nodejs, test it with an http trigger
echo "Poolmgr ExecutorType: Pre-test cleanup"
log "Poolmgr ExecutorType: Pre-test cleanup"
fission env delete --name nodejs || true
echo "Creating nodejs env"
log "Creating nodejs env"
fission env create --name nodejs --image fission/node-env --mincpu 20 --maxcpu 100 --minmemory 128 --maxmemory 256
trap "fission env delete --name nodejs" EXIT
echo "Creating function"
log "Creating function"
fission fn create --name $fn --env nodejs --code $ROOT/examples/nodejs/hello.js --executortype poolmgr
trap "fission fn delete --name $fn" EXIT
echo "Creating route"
log "Creating route"
fission route create --function $fn --url /$fn --method GET
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 5
echo "Doing an HTTP GET on the function's route"
log "Doing an HTTP GET on the function's route"
response=$(curl http://$FISSION_ROUTER/$fn)
echo "Checking for valid response"
log "Checking for valid response"
echo $response | grep -i hello
# crappy cleanup, improve this later
kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
echo "Poolmgr ExecutorType: All done."
log "Poolmgr ExecutorType: All done."
test/tests/test_buildermgr.sh
+13 -13
View File
@@ -15,16 +15,16 @@ PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test
fn=python-srcbuild-$(date +%s)
checkFunctionResponse() {
echo "Doing an HTTP GET on the function's route"
log "Doing an HTTP GET on the function's route"
response=$(curl http://$FISSION_ROUTER/$1)
echo "Checking for valid response"
echo $response
log "Checking for valid response"
log $response
echo $response | grep -i "a: 1 b: {c: 3, d: 4}"
}
waitBuild() {
echo "Waiting for builder manager to finish the build"
log "Waiting for builder manager to finish the build"
while true; do
kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded
@@ -39,7 +39,7 @@ waitEnvBuilder() {
env=$1
envRV=$(kubectl -n default get environments ${env} -o jsonpath='{.metadata.resourceVersion}')
echo "Waiting for env builder to catch up"
log "Waiting for env builder to catch up"
while true; do
kubectl -n fission-builder get pod -l envName=${env},envResourceVersion=${envRV} \
@@ -51,27 +51,27 @@ waitEnvBuilder() {
}
export -f waitEnvBuilder
echo "Pre-test cleanup"
log "Pre-test cleanup"
fission env delete --name python || true
kubectl --namespace default get packages|grep -v NAME|awk '{print $1}'|xargs -I@ bash -c 'kubectl --namespace default delete packages @' || true
echo "Creating python env"
log "Creating python env"
fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
trap "fission env delete --name python" EXIT
timeout 180s bash -c "waitEnvBuilder python"
echo "Creating source pacakage"
log "Creating source pacakage"
zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
echo "Creating function " $fn
log "Creating function " $fn
fission fn create --name $fn --env python --src demo-src-pkg.zip --entrypoint "user.main" --buildcmd "./build.sh"
trap "fission fn delete --name $fn" EXIT
echo "Creating route"
log "Creating route"
fission route create --function $fn --url /$fn --method GET
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 3
pkg=$(kubectl --namespace default get functions $fn -o jsonpath='{.spec.package.packageref.name}')
@@ -81,7 +81,7 @@ timeout 60s bash -c "waitBuild $pkg"
checkFunctionResponse $fn
echo "Updating function " $fn
log "Updating function " $fn
fission fn update --name $fn --src demo-src-pkg.zip
trap "fission fn delete --name $fn" EXIT
@@ -95,4 +95,4 @@ checkFunctionResponse $fn
# crappy cleanup, improve this later
kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
echo "All done."
log "All done."
test/tests/test_function_update.sh
+12 -12
View File
@@ -10,28 +10,28 @@ fn=nodejs-hello-$(date +%s)
# Update it and check it's output, the output should be
# different from the previous one.
echo "Pre-test cleanup"
log "Pre-test cleanup"
fission env delete --name nodejs || true
echo "Creating nodejs env"
log "Creating nodejs env"
fission env create --name nodejs --image fission/node-env
trap "fission env delete --name nodejs" EXIT
echo "Creating function"
log "Creating function"
echo 'module.exports = function(context, callback) { callback(200, "foo!\n"); }' > foo.js
fission fn create --name $fn --env nodejs --code foo.js
trap "fission fn delete --name $fn" EXIT
echo "Creating route"
log "Creating route"
fission route create --function $fn --url /$fn --method GET
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 10
echo "Doing an HTTP GET on the function's route"
log "Doing an HTTP GET on the function's route"
response=$(curl http://$FISSION_ROUTER/$fn)
echo "Checking for valid response"
log "Checking for valid response"
echo $response | grep -i foo
# Running a background process to keep access the
@@ -40,18 +40,18 @@ echo $response | grep -i foo
( watch -n1 curl http://$FISSION_ROUTER/$fn ) > /dev/null 2>&1 &
pid=$!
echo "Updating function"
log "Updating function"
echo 'module.exports = function(context, callback) { callback(200, "bar!\n"); }' > bar.js
fission fn update --name $fn --code bar.js
trap "fission fn delete --name $fn" EXIT
echo "Waiting for router to update cache"
log "Waiting for router to update cache"
sleep 10
echo "Doing an HTTP GET on the function's route"
log "Doing an HTTP GET on the function's route"
response=$(curl http://$FISSION_ROUTER/$fn)
echo "Checking for valid response again"
log "Checking for valid response again"
echo $response | grep -i bar
kill -15 $pid
@@ -59,4 +59,4 @@ kill -15 $pid
# crappy cleanup, improve this later
kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
echo "All done."
log "All done."
test/tests/test_internal_routes.sh
+8 -8
View File
@@ -9,38 +9,38 @@ set -euo pipefail
ROOT=$(dirname $0)/../..
echo "Pre-test cleanup"
log "Pre-test cleanup"
fission env delete --name nodejs || true
echo "Creating nodejs env"
log "Creating nodejs env"
fission env create --name nodejs --image fission/node-env
trap "fission env delete --name nodejs" EXIT
echo "Writing functions"
log "Writing functions"
f1=f1-$(date +%s)
f2=f2-$(date +%s)
echo $f1 $f2
log $f1 $f2
for f in $f1 $f2
do
echo "module.exports = function(context, callback) { callback(200, \"$f\n\"); }" > $f.js
done
echo "Creating functions"
log "Creating functions"
for f in $f1 $f2
do
fission fn create --name $f --env nodejs --code $f.js
trap "fission fn delete --name $f" EXIT
done
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 2
echo "Testing internal routes"
log "Testing internal routes"
for f in $f1 $f2
do
response=$(curl http://$FISSION_ROUTER/fission-function/$f)
echo $response | grep $f
done
echo "All done."
log "All done."
test/tests/test_logging/test_function_logs.sh
+15 -15
View File
@@ -8,39 +8,39 @@ ROOT=$(dirname $0)/../..
fn=nodejs-logtest-$(date +%N)
function cleanup {
echo "Cleanup route"
log "Cleanup route"
var=$(fission route list | grep $fn | awk '{print $1;}')
fission route delete --name $var
echo "delete logfile"
log "delete logfile"
rm "/tmp/logfile"
}
# Create a hello world function in nodejs, test it with an http trigger
echo "Pre-test cleanup"
log "Pre-test cleanup"
fission env delete --name nodejs || true
echo "Creating nodejs env"
log "Creating nodejs env"
fission env create --name nodejs --image fission/node-env
trap "fission env delete --name nodejs" EXIT
echo "Creating function"
log "Creating function"
fission fn create --name $fn --env nodejs --code log.js
trap "fission fn delete --name $fn" EXIT
echo "Creating route"
log "Creating route"
fission route create --function $fn --url /$fn --method GET
trap cleanup EXIT
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 15
echo "Doing 4 HTTP GETs on the function's route"
log "Doing 4 HTTP GETs on the function's route"
for i in 1 2 3 4
do
curl -s http://$FISSION_ROUTER/$fn
done
echo "Grabbing logs, should have 4 calls in logs"
log "Grabbing logs, should have 4 calls in logs"
sleep 15
@@ -52,15 +52,15 @@ then
fission function logs --name $fn --detail > /tmp/logfile
fi
echo "---function logs---"
log "---function logs---"
cat /tmp/logfile
echo "------"
log "------"
num=$(grep 'log test' /tmp/logfile | wc -l)
echo $num logs found
log $num logs found
if [ $num -ne 4 ]
then
echo "Test Failed: expected 4, found $num logs"
log "Test Failed: expected 4, found $num logs"
fi
echo "All done."
log "All done."
test/tests/test_node_hello_http.sh
+8 -8
View File
@@ -7,30 +7,30 @@ ROOT=$(dirname $0)/../..
fn=nodejs-hello-$(date +%N)
# Create a hello world function in nodejs, test it with an http trigger
echo "Pre-test cleanup"
log "Pre-test cleanup"
fission env delete --name nodejs || true
echo "Creating nodejs env"
log "Creating nodejs env"
fission env create --name nodejs --image fission/node-env
trap "fission env delete --name nodejs" EXIT
echo "Creating function"
log "Creating function"
fission fn create --name $fn --env nodejs --code $ROOT/examples/nodejs/hello.js
trap "fission fn delete --name $fn" EXIT
echo "Creating route"
log "Creating route"
fission route create --function $fn --url /$fn --method GET
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 3
echo "Doing an HTTP GET on the function's route"
log "Doing an HTTP GET on the function's route"
response=$(curl http://$FISSION_ROUTER/$fn)
echo "Checking for valid response"
log "Checking for valid response"
echo $response | grep -i hello
# crappy cleanup, improve this later
kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
echo "All done."
log "All done."
test/tests/test_package_command.sh
+15 -15
View File
@@ -14,7 +14,7 @@ PYTHON_BUILDER_IMAGE=gcr.io/fission-ci/python-env-builder:test
fn=python-srcbuild-$(date +%s)
waitBuild() {
echo "Waiting for builder manager to finish the build"
log "Waiting for builder manager to finish the build"
while true; do
kubectl --namespace default get packages $1 -o jsonpath='{.status.buildstatus}'|grep succeeded
@@ -26,11 +26,11 @@ waitBuild() {
export -f waitBuild
checkFunctionResponse() {
echo "Doing an HTTP GET on the function's route"
log "Doing an HTTP GET on the function's route"
response=$(curl http://$FISSION_ROUTER/$1)
echo "Checking for valid response"
echo $response
log "Checking for valid response"
log $response
echo $response | grep -i "$2"
}
@@ -38,7 +38,7 @@ waitEnvBuilder() {
env=$1
envRV=$(kubectl -n default get environments ${env} -o jsonpath='{.metadata.resourceVersion}')
echo "Waiting for env builder to catch up"
log "Waiting for env builder to catch up"
while true; do
kubectl -n fission-builder get pod -l envName=${env},envResourceVersion=${envRV} \
@@ -50,46 +50,46 @@ waitEnvBuilder() {
}
export -f waitEnvBuilder
echo "Pre-test cleanup"
log "Pre-test cleanup"
fission env delete --name python || true
echo "Creating python env"
log "Creating python env"
fission env create --name python --image $PYTHON_RUNTIME_IMAGE --builder $PYTHON_BUILDER_IMAGE
trap "fission env delete --name python" EXIT
timeout 180s bash -c "waitEnvBuilder python"
echo "Creating pacakage with source archive"
log "Creating pacakage with source archive"
zip -jr demo-src-pkg.zip $ROOT/examples/python/sourcepkg/
pkgName=$(fission package create --src demo-src-pkg.zip --env python --buildcmd "./build.sh"| cut -f2 -d' '| tr -d \')
# wait for build to finish at most 60s
timeout 60s bash -c "waitBuild $pkgName"
echo "Creating function " $fn
log "Creating function " $fn
fission fn create --name $fn --pkg $pkgName --entrypoint "user.main"
trap "fission fn delete --name $fn" EXIT
echo "Creating route"
log "Creating route"
fission route create --function $fn --url /$fn --method GET
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 3
checkFunctionResponse $fn 'a: 1 b: {c: 3, d: 4}'
echo "Creating package with deploy archive"
log "Creating package with deploy archive"
mkdir testDir
touch testDir/__init__.py
printf 'def main():\n return "Hello, world!"' > testDir/hello.py
zip -jr demo-deploy-pkg.zip testDir/
pkgName=$(fission package create --deploy demo-deploy-pkg.zip --env python| cut -f2 -d' '| tr -d \')
echo "Updating function " $fn
log "Updating function " $fn
fission fn update --name $fn --pkg $pkgName --entrypoint "hello.main"
trap "fission fn delete --name $fn" EXIT
echo "Waiting for router to update cache"
log "Waiting for router to update cache"
sleep 3
checkFunctionResponse $fn 'Hello, world!'
@@ -97,4 +97,4 @@ checkFunctionResponse $fn 'Hello, world!'
# crappy cleanup, improve this later
kubectl get httptrigger -o name | tail -1 | cut -f2 -d'/' | xargs kubectl delete httptrigger
echo "All done."
log "All done."
test/tests/test_pass.sh
+3 -3
View File
@@ -5,8 +5,8 @@ set -euo pipefail
# This doesn't test fission, just the test framework. It ensures we
# have the right environment, that's all.
echo "Test test, please ignore."
log "Test test, please ignore."
echo $FISSION_URL
echo $FISSION_ROUTER
log $FISSION_URL
log $FISSION_ROUTER
which fission
test/tests/test_secret_cfgmap/test_secret_cfgmap.sh
+24 -24
View File
@@ -15,7 +15,7 @@ cp cfgmap.py.template cfgmap.py
sed -i "s/{{ FN_CFGMAP }}/${fn_cfgmap}/g" cfgmap.py
function cleanup {
echo "Cleanup everything"
log "Cleanup everything"
kubectl delete secret -n default ${fn_secret}
kubectl delete configmap -n default ${fn_cfgmap}
fission function delete --name ${fn_secret}
@@ -30,84 +30,84 @@ function cleanup {
}
# Create a hello world function in nodejs, test it with an http trigger
echo "Pre-test cleanup"
log "Pre-test cleanup"
fission env delete --name python || true
echo "Creating python env"
log "Creating python env"
fission env create --name python --image fission/python-env
trap "fission env delete --name python" EXIT
echo "Creating secret"
log "Creating secret"
kubectl create secret generic ${fn_secret} --from-literal=TEST_KEY="TESTVALUE" -n default
trap "kubectl delete secret ${fn_secret} -n default" EXIT
echo "Creating function with secret"
log "Creating function with secret"
fission fn create --name ${fn_secret} --env python --code secret.py --secret ${fn_secret}
trap "fission fn delete --name ${fn_secret}" EXIT
echo "Creating route"
log "Creating route"
fission route create --function ${fn_secret} --url /${fn_secret} --method GET
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 5
echo "HTTP GET on the function's route"
log "HTTP GET on the function's route"
res=$(curl http://${FISSION_ROUTER}/${fn_secret})
val='TESTVALUE'
if [[ ${res} != ${val} ]]
then
echo "test secret failed"
log "test secret failed"
cleanup
exit 1
fi
echo "test secret passed"
log "test secret passed"
echo "Creating configmap"
log "Creating configmap"
kubectl create configmap ${fn_cfgmap} --from-literal=TEST_KEY=TESTVALUE -n default
trap "kubectl delete configmap ${fn_cfgmap} -n default" EXIT
echo "creating function with configmap"
log "creating function with configmap"
fission fn create --name ${fn_cfgmap} --env python --code cfgmap.py --configmap ${fn_cfgmap}
trap "fission fn delete --name ${fn_cfgmap}" EXIT
echo "Creating route"
log "Creating route"
fission route create --function ${fn_cfgmap} --url /${fn_cfgmap} --method GET
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 5
echo "HTTP GET on the function's route"
log "HTTP GET on the function's route"
rescfg=$(curl http://${FISSION_ROUTER}/${fn_cfgmap})
if [ ${rescfg} != ${val} ]
then
echo "test cfgmap failed"
log "test cfgmap failed"
cleanup
exit 1
fi
echo "test configmap passed"
log "test configmap passed"
echo "testing creating a function without a secret or configmap"
log "testing creating a function without a secret or configmap"
fission function create --name ${fn} --env python --code empty.py
trap "fission fn delete --name ${fn}" EXIT
echo "Creating route"
log "Creating route"
fission route create --function ${fn} --url /${fn} --method GET
echo "Waiting for router to catch up"
log "Waiting for router to catch up"
sleep 5
echo "HTTP GET on the function's route"
log "HTTP GET on the function's route"
resnormal=$(curl http://${FISSION_ROUTER}/${fn})
if [ ${resnormal} != "yes" ]
then
echo "test empty failed"
log "test empty failed"
cleanup
exit 1
fi
echo "test empty passed"
log "test empty passed"
echo "All done."
log "All done."
trap "cleanup" EXIT