Storage service helm chart integration + bugfixes (#315)

Add storage service and a persistent volume claim to helm charts. A few bugfixes for the api proxy.

charts/fission-all/templates/deployment.yaml

@@ -484,3 +484,36 @@ spec:
         - name: MESSAGE_QUEUE_URL
           value: nats://{{ .Values.nats.authToken }}@nats-streaming:4222
       serviceAccount: fission-svc
+
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: storagesvc
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        svc: storagesvc
+    spec:
+      containers:
+      - name: storagesvc
+        image: "{{ .Values.image }}:{{ .Values.imageTag }}"
+        imagePullPolicy: {{ .Values.pullPolicy }}
+        command: ["/fission-bundle"]
+        args: ["--storageServicePort", "8000", "--filePath", "/fission"]
+        volumeMounts:
+        - name: fission-storage
+          mountPath: /fission
+      serviceAccount: fission-svc
+      volumes:
+      - name: fission-storage
+      {{- if .Values.persistence.enabled }}   
+        persistentVolumeClaim:
+          claimName: {{ .Values.persistence.existingClaim | default "fission-storage-pvc" }}
+      {{- else }}
+        emptyDir: {}
+      {{- end -}}

charts/fission-all/templates/pvc.yaml

@@ -0,0 +1,23 @@
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: fission-storage-pvc
+  labels:
+    app: fission-storage
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+  annotations:
+  {{- if .Values.persistence.storageClass }}
+    volume.beta.kubernetes.io/storage-class: {{ .Values.persistence.storageClass | quote }}
+  {{- else }}
+    volume.alpha.kubernetes.io/storage-class: default
+  {{- end }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size | quote }}
+
+{{- end }}

charts/fission-all/templates/svc.yaml

@@ -65,3 +65,18 @@ spec:
   selector:
     svc: nats-streaming
 
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: storagesvc
+  labels:
+    svc: storagesvc
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+spec:
+  type: ClusterIP
+  ports:
+  - port: 80
+    targetPort: 8000
+  selector:
+    svc: storagesvc

charts/fission-all/values.yaml

@@ -48,3 +48,13 @@ nats:
   authToken: "defaultFissionAuthToken"
   clusterID: "fissionMQTrigger"
   
+
+## Persist data to a persistent volume.
+persistence:
+  enabled: true
+  ## If defined, volume.beta.kubernetes.io/storage-class: <storageClass>
+  ## Default: volume.alpha.kubernetes.io/storage-class: default
+  ##
+  # storageClass:
+  accessMode: ReadWriteOnce
+  size: 8Gi

charts/fission-core/templates/deployment.yaml

@@ -285,3 +285,36 @@ spec:
         command: ["/fission-bundle"]
         args: ["--timer"]
       serviceAccount: fission-svc
+
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: storagesvc
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        svc: storagesvc
+    spec:
+      containers:
+      - name: storagesvc
+        image: "{{ .Values.image }}:{{ .Values.imageTag }}"
+        imagePullPolicy: {{ .Values.pullPolicy }}
+        command: ["/fission-bundle"]
+        args: ["--storageServicePort", "8000", "--filePath", "/fission"]
+        volumeMounts:
+        - name: fission-storage
+          mountPath: /fission
+      serviceAccount: fission-svc
+      volumes:
+      - name: fission-storage
+      {{- if .Values.persistence.enabled }}   
+        persistentVolumeClaim:
+          claimName: {{ .Values.persistence.existingClaim | default "fission-storage-pvc" }}
+      {{- else }}
+        emptyDir: {}
+      {{- end -}}

charts/fission-core/templates/pvc.yaml

@@ -0,0 +1,23 @@
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: fission-storage-pvc
+  labels:
+    app: fission-storage
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+  annotations:
+  {{- if .Values.persistence.storageClass }}
+    volume.beta.kubernetes.io/storage-class: {{ .Values.persistence.storageClass | quote }}
+  {{- else }}
+    volume.alpha.kubernetes.io/storage-class: default
+  {{- end }}
+spec:
+  accessModes:
+    - {{ .Values.persistence.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ .Values.persistence.size | quote }}
+
+{{- end }}

charts/fission-core/templates/svc.yaml

@@ -29,4 +29,20 @@ spec:
     targetPort: 8888
     nodePort: {{ .Values.controllerPort }}
   selector:
-    svc: controller
+    svc: controller
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: storagesvc
+  labels:
+    svc: storagesvc
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+spec:
+  type: ClusterIP
+  ports:
+  - port: 80
+    targetPort: 8000
+  selector:
+    svc: storagesvc

charts/fission-core/values.yaml

@@ -7,19 +7,19 @@
 serviceType: LoadBalancer
 
 ## Fission image repository
-image: fission/fission-bundle
+image: minikube/fission-bundle
 
 ## Fission image version
-imageTag: v0.2.0-20170822
+imageTag: testing
 
 ## Image pull policy
 pullPolicy: IfNotPresent
 
 ## Fission fetcher repository
-fetcherImage: fission/fetcher
+fetcherImage: minikube/fetcher
 
 ## Fission fetcher image version
-fetcherImageTag: v0.2.0-20170822
+fetcherImageTag: testing
 
 ## Port at which Fission controller service should be exposed
 controllerPort: 31313
@@ -33,3 +33,13 @@ functionNamespace: fission-function
 
 ## Set up openshift RBAC rule
 openshift: false
+  
+## Persist data to a persistent volume.
+persistence:
+  enabled: true
+  ## If defined, volume.beta.kubernetes.io/storage-class: <storageClass>
+  ## Default: volume.alpha.kubernetes.io/storage-class: default
+  ##
+  # storageClass:
+  accessMode: ReadWriteOnce
+  size: 8Gi

controller/storagesvc.go

@@ -25,7 +25,7 @@ import (
 )
 
 func (api *API) StorageServiceProxy(w http.ResponseWriter, r *http.Request) {
-	u := api.storageServiceUrl + "/v1/archive"
+	u := api.storageServiceUrl
 	ssUrl, err := url.Parse(u)
 	if err != nil {
 		msg := fmt.Sprintf("Error parsing url %v: %v", u, err)
@@ -33,6 +33,13 @@ func (api *API) StorageServiceProxy(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, msg, 500)
 		return
 	}
-	proxy := httputil.NewSingleHostReverseProxy(ssUrl)
+	director := func(req *http.Request) {
+		req.URL.Scheme = ssUrl.Scheme
+		req.URL.Host = ssUrl.Host
+		req.URL.Path = "/v1/archive"
+	}
+	proxy := &httputil.ReverseProxy{
+		Director: director,
+	}
 	proxy.ServeHTTP(w, r)
 }

fission-bundle/main.go

@@ -173,7 +173,7 @@ Options:
 		runMessageQueueMgr(routerUrl)
 	}
 
-	if arguments["--storageServicePort"] == true {
+	if arguments["--storageServicePort"] != nil {
 		port := getPort(arguments["--storageServicePort"])
 		filePath := arguments["--filePath"].(string)
 		runStorageSvc(port, filePath)

storagesvc/client/storagesvc_test.go

@@ -96,5 +96,5 @@ func TestStorageService(t *testing.T) {
 	}
 
 	// cleanup /tmp
-	os.RemoveAll(fmt.Sprintf("/tmp/", testId))
+	os.RemoveAll(fmt.Sprintf("/tmp/%v", testId))
 }

storagesvc/storagesvc.go

@@ -132,6 +132,7 @@ func (ss *StorageService) deleteHandler(w http.ResponseWriter, r *http.Request)
 	fileId, err := ss.getIdFromRequest(r)
 	if err != nil {
 		http.Error(w, err.Error(), 400)
+		return
 	}
 
 	err = ss.container.RemoveItem(fileId)
@@ -148,6 +149,7 @@ func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request
 	fileId, err := ss.getIdFromRequest(r)
 	if err != nil {
 		http.Error(w, err.Error(), 400)
+		return
 	}
 
 	// Get the file (called "item" in stow's jargon), open it,