S3 backend for storage service (#1629)

Co-authored-by: Alok Kumar <rajalokan@gmail.com>

charts/fission-all/templates/deployment.yaml

@@ -776,7 +776,11 @@ spec:
         image: {{ include "fission-bundleImage" . | quote }}
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
-        args: ["--storageServicePort", "8000", "--filePath", "/fission"]
+        {{- if and (.Values.persistence.enabled) (eq (.Values.persistence.storageType | default "local") "s3") }}
+        args: ["--storageServicePort", "8000", "--storageType", "s3"]
+        {{- else }}
+        args: ["--storageServicePort", "8000", "--storageType", "local"]
+        {{- end }}
         env:
         - name: TRACE_JAEGER_COLLECTOR_ENDPOINT
           value: "{{ .Values.traceCollectorEndpoint }}"
@@ -786,9 +790,25 @@ spec:
           value: "{{.Values.pruneInterval}}"
         - name: DEBUG_ENV
           value: {{ .Values.debugEnv | quote }}
+        {{- if and (.Values.persistence.enabled) (eq (.Values.persistence.storageType | default "local") "s3") }}
+        - name: STORAGE_S3_ENDPOINT
+          value: {{ .Values.persistence.s3.endPoint }}
+        - name: STORAGE_S3_BUCKET_NAME
+          value: {{ .Values.persistence.s3.bucketName }}
+        - name: STORAGE_S3_SUB_DIR
+          value: {{ .Values.persistence.s3.subDir }}
+        - name: STORAGE_S3_ACCESS_KEY_ID
+          value: {{ .Values.persistence.s3.accessKeyId }}
+        - name: STORAGE_S3_SECRET_ACCESS_KEY
+          value: {{ .Values.persistence.s3.secretAccessKey }}
+        - name: STORAGE_S3_REGION
+          value: {{ .Values.persistence.s3.region }}
+        {{- end }}
+        {{- if ne (.Values.persistence.storageType | default "local") "s3" }}
         volumeMounts:
         - name: fission-storage
           mountPath: /fission
+        {{- end }}
         readinessProbe:
           httpGet:
             path: "/healthz"
@@ -806,12 +826,14 @@ spec:
           - containerPort: 8000
             name: http
       serviceAccountName: fission-svc
+      {{- if and (.Values.persistence.enabled) (ne (.Values.persistence.storageType | default "local") "s3") }}
       volumes:
       - name: fission-storage
-      {{- if .Values.persistence.enabled }}
         persistentVolumeClaim:
           claimName: {{ .Values.persistence.existingClaim | default "fission-storage-pvc" }}
       {{- else }}
+      volumes:
+      - name: fission-storage
         emptyDir: {}
       {{- end }}
 {{- if .Values.extraCoreComponentPodConfig }}

charts/fission-all/templates/pvc.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+{{- if and (.Values.persistence.enabled) (not .Values.persistence.existingClaim) (eq (.Values.persistence.storageType | default "local") "local") }}
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:

charts/fission-all/values.yaml

@@ -203,11 +203,23 @@ kafka:
 
 ## Persist data to a persistent volume.
 persistence:
-  ## If true, fission will create/use a Persistent Volume Claim
+  ## If true, fission will create/use a Persistent Volume Claim unless storageType is set to s3
   ## If false, use emptyDir
   ##
   enabled: true
 
+  ## Must be set to either local or S3.
+  ## If storateType is set(other than local), one of its backend configuration must be set as below.
+  #storageType: local | s3
+
+  ## Sample configruation for AWS s3 storage backend
+  #s3:
+  # bucketName: <awsBucketName>
+  # subDir: <sub directory within a bucket>
+  # accessKeyId: <awsAccessKeyId>
+  # secretAccessKey: <awsSecretAccessKey>
+  # region: <awsRegion>
+
   ## A manually managed Persistent Volume Claim name
   ## Requires persistence.enabled: true
   ## If defined, PVC must be created manually before volume will be bound

charts/fission-core/templates/deployment.yaml

@@ -288,7 +288,7 @@ spec:
         - name: TRACE_JAEGER_COLLECTOR_ENDPOINT
           value: "{{ .Values.traceCollectorEndpoint }}"
         - name: TRACING_SAMPLING_RATE
-          value: {{ .Values.traceSamplingRate | default "0.5" | quote }}          
+          value: {{ .Values.traceSamplingRate | default "0.5" | quote }}      
         - name: ENABLE_ISTIO
           value: "{{ .Values.enableIstio }}"
         - name: FETCHER_MINCPU
@@ -398,27 +398,49 @@ spec:
         image: {{ include "fission-bundleImage" . | quote }}
         imagePullPolicy: {{ .Values.pullPolicy }}
         command: ["/fission-bundle"]
-        args: ["--storageServicePort", "8000", "--filePath", "/fission"]
+        {{- if and (.Values.persistence.enabled) (eq (.Values.persistence.storageType | default "local") "s3") }}
+        args: ["--storageServicePort", "8000", "--storageType", "s3"]
+        {{- else }}
+        args: ["--storageServicePort", "8000", "--storageType", "local"]
+        {{- end }}
         env:
         - name: PRUNE_INTERVAL
           value: "{{.Values.pruneInterval}}"
         - name: TRACE_JAEGER_COLLECTOR_ENDPOINT
           value: "{{ .Values.traceCollectorEndpoint }}"
         - name: TRACING_SAMPLING_RATE
-          value: {{ .Values.traceSamplingRate | default "0.5" | quote }}          
+          value: {{ .Values.traceSamplingRate | default "0.5" | quote }}
+        {{- if and (.Values.persistence.enabled) (eq (.Values.persistence.storageType | default "local") "s3") }}
+        - name: STORAGE_S3_ENDPOINT
+          value: {{ .Values.persistence.s3.endPoint }}
+        - name: STORAGE_S3_BUCKET_NAME
+          value: {{ .Values.persistence.s3.bucketName }}
+        - name: STORAGE_S3_SUB_DIR
+          value: {{ .Values.persistence.s3.subDir }}
+        - name: STORAGE_S3_ACCESS_KEY_ID
+          value: {{ .Values.persistence.s3.accessKeyId }}
+        - name: STORAGE_S3_SECRET_ACCESS_KEY
+          value: {{ .Values.persistence.s3.secretAccessKey }}
+        - name: STORAGE_S3_REGION
+          value: {{ .Values.persistence.s3.region }}
+        {{- end }}
+        {{- if ne (.Values.persistence.storageType | default "local") "s3" }}
         volumeMounts:
         - name: fission-storage
           mountPath: /fission
+        {{- end }}
         ports:
           - containerPort: 8000
             name: http
       serviceAccountName: fission-svc
+      {{- if and (.Values.persistence.enabled) (ne (.Values.persistence.storageType | default "local") "s3") }}
       volumes:
       - name: fission-storage
-      {{- if .Values.persistence.enabled }}   
         persistentVolumeClaim:
           claimName: {{ .Values.persistence.existingClaim | default "fission-storage-pvc" }}
       {{- else }}
+      volumes:
+      - name: fission-storage
         emptyDir: {}
       {{- end }}
 {{- if .Values.extraCoreComponentPodConfig }}

charts/fission-core/templates/pvc.yaml

@@ -1,4 +1,4 @@
-{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+{{- if and (.Values.persistence.enabled) (not .Values.persistence.existingClaim) (eq (.Values.persistence.storageType | default "local") "local") }}
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:

charts/fission-core/values.yaml

@@ -110,11 +110,23 @@ router:
 
 ## Persist data to a persistent volume.
 persistence:
-  ## If true, fission will create/use a Persistent Volume Claim
+  ## If true, fission will create/use a Persistent Volume Claim unless storageType is set to s3
   ## If false, use emptyDir
   ##
   enabled: true
 
+  ## Must be set to either local or S3.
+  ## If storateType is set(other than local), one of its backend configuration must be set as below.
+  #storageType: local | s3
+
+  ## Sample configruation for AWS s3 storage backend
+  #s3:
+  # bucketName: <awsBucketName>
+  # subDir: <sub directory within a bucket>
+  # accessKeyId: <awsAccessKeyId>
+  # secretAccessKey: <awsSecretAccessKey>
+  # region: <awsRegion>
+
   ## A manually managed Persistent Volume Claim name
   ## Requires persistence.enabled: true
   ## If defined, PVC must be created manually before volume will be bound