Migrate to Helm

build/Jenkinsfile

@@ -11,7 +11,7 @@ node("docker") {
             stage("Prepare environments") {
                 if (env.BRANCH_NAME == 'master') 
                     env.stand = "prod"
-                else if (env.BRANCH_NAME == 'dev') 
+                else if (env.BRANCH_NAME == 'dev')
                     env.stand = "test"
                 else
                     error("Unsupported branch: ${env.BRANCH_NAME}")
@@ -19,9 +19,11 @@ node("docker") {
                 envConfig = "build/vars/build.yaml"
                 support.getVarsV2(envConfig)
 
-                // временно для этого микросервиса
+                // пока не будет готов отдельный dev стенд
-                if (env.BRANCH_NAME == 'dev')
+                if (env.BRANCH_NAME == 'dev') {
-                    env.namespace = "${env.namespace}-test"
+                    env.namespace             = "${env.namespace}-test"
+                    env.registryContainerName = "${env.registryContainerName}-test" 
+                }
 
                 println "Success prepare"
             }
@@ -41,7 +43,7 @@ node("docker") {
 
             stage("Update version in Kubernetes") {
                 dir ("build") {
-                    withCredentials([vaultString(credentialsId: "config.${env.registryContainerName}-${env.stand}", variable: "pem")]) {
+                    withCredentials([vaultString(credentialsId: "${env.vaultPath}", variable: "pem")]) {
                         sh """
                         set +x
                         echo "${pem}" | base64 -d > auth.config
@@ -49,10 +51,18 @@ node("docker") {
                         """
                     }
 
-                    convertJinja("microservice.yaml.tmpl", "microservice.yaml")
+                    convertJinjaV2("./helm/values.yaml", debug=true)
+
+                    sh """
+                        helm upgrade --wait \
+                        --kubeconfig auth.config \
+                        --install \
+                        --namespace ${env.namespace} \
+                        -f ./helm/values.yaml \
+                        ${env.appName} ./helm/
+                    """
                     
                     sh "kubectl --kubeconfig=auth.config get pods --namespace ${env.namespace}"
-                    sh "kubectl --kubeconfig=auth.config apply -f microservice.yaml"
                 }
             } 
             

build/helm/Chart.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v2
+type: application
+name: svc-api
+description: Microservice "svc-api" for deck
+
+version: 0.0.1
+appVersion: 0.0.1

build/helm/templates/configmap.yaml

@@ -0,0 +1,114 @@
+{{- if .Values.sidecars.logger.enabled  }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "fluentbit-{{ .Chart.Name }}"
+data:
+  fluent-bit.conf: |
+    [SERVICE]
+        Parsers_File    /fluent-bit/etc/parsers.conf
+
+    [INPUT]
+        Name                tail
+        Tag                 lk
+        Path                /usr/lib/serverHome/logs/access.txt*
+        Mem_Buf_Limit       100MB
+        Skip_Long_Lines     On
+        Refresh_Interval    5
+
+    [FILTER]
+        Name record_modifier
+        Match *
+        Whitelist_key log
+
+    [FILTER]
+        Name rewrite_tag
+        Match {{ .Chart.Name }}
+        Rule $log kube-probe {{ .Chart.Name }}-healtcheck false
+
+    [FILTER]
+        Name rewrite_tag
+        Match {{ .Chart.Name }}
+        Rule $log {{ .Chart.Name }} {{ .Chart.Name }}.access false
+
+    [FILTER]
+        Name parser
+        Match {{ .Chart.Name }}.access
+        Key_name log
+        Parser lucee_parser
+
+    [OUTPUT]
+        Name        loki
+        Match       *
+        Host        10.96.52.75
+        labels      job={{ .Chart.Name }}-fluentbit-{{ .Values.stand }},$kubernetes['pod_name'],$kubernetes['namespace_name'],$kubernetes['host'],$kubernetes['container_name']
+        port        3100
+        auto_kubernetes_labels on
+
+    [OUTPUT]
+        Name stdout
+        Match *
+
+  parsers.conf: |-
+    [PARSER]
+      Name lucee_parser
+      Format regex
+      Regex ^(?<remote_address>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+      Types status:integer
+
+    [PARSER]
+        Name   apache
+        Format regex
+        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   apache2
+        Format regex
+        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   apache_error
+        Format regex
+        Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
+
+    [PARSER]
+        Name   nginx
+        Format regex
+        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   json
+        Format json
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name        docker
+        Format      json
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        # http://rubular.com/r/tjUt3Awgg4
+        Name cri
+        Format regex
+        Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+
+    [PARSER]
+        Name        syslog
+        Format      regex
+        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
+        Time_Key    time
+        Time_Format %b %d %H:%M:%S
+
+{{- end }}

build/helm/templates/deployment.yaml

@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: "{{ .Chart.Name }}"
+    stand: "{{ .Values.stand }}"
+  name: "{{ .Chart.Name }}"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: "{{ .Chart.Name }}"
+      stand: "{{ .Values.stand }}"
+  template:
+    metadata:
+      labels:
+        app: "{{ .Chart.Name }}"
+        stand: "{{ .Values.stand }}"
+    spec:
+      {{- if .Values.vault.enabled }}
+      serviceAccountName: "{{ .Values.vault.serviceAccountName }}"
+      {{- end }}
+      containers:
+        # Основное приложение
+        - name: app
+          image: "{{ .Values.mainContainer.image.repository }}:{{ .Values.mainContainer.image.tag }}"
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: {{ .Values.mainContainer.containerPort }}
+              protocol: TCP
+          resources:
+            {{- toYaml .Values.mainContainer.resources | nindent 12 }}
+          readinessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 2
+            {{- toYaml .Values.mainContainer.readinessProbe | nindent 12 }}
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+            {{- toYaml .Values.mainContainer.livenessProbe | nindent 12 }}
+          {{- if .Values.vault.enabled }}
+          envFrom:
+            - secretRef:
+                name: "{{ .Values.vault.target.name }}"
+          {{- end }}
+          volumeMounts:
+            {{- toYaml .Values.mainContainer.volumeMounts | nindent 12 }}
+          
+        # Sidecar-контейнеры
+        {{- if .Values.sidecars.logger.enabled }}
+        - name: logger
+          image: "{{ .Values.sidecars.logger.image.repository }}:{{ .Values.sidecars.logger.image.tag }}"
+          imagePullPolicy: IfNotPresent
+          resources:
+            {{- toYaml .Values.sidecars.logger.resources | nindent 12 }}
+          volumeMounts:
+            - name: fluentbit-config
+              mountPath: /fluent-bit/etc/
+            {{- toYaml .Values.sidecars.logger.volumeMounts | nindent 12 }}
+        {{- end }}
+      
+      volumes:
+        {{- if .Values.sidecars.logger.enabled }}
+        - name: fluentbit-config
+          configMap:
+            name: fluentbit-{{ .Chart.Name }}
+        {{- end }}
+        {{- toYaml .Values.mainContainer.volumes | nindent 8 }}

build/helm/templates/service.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ .Chart.Name }}"
+  labels:
+    app: "{{ .Chart.Name }}"
+    stand: "{{ .Values.stand }}"
+spec:
+  ports:
+  - port: {{ .Values.mainContainer.containerPort }}
+    targetPort: {{ .Values.mainContainer.containerPort }}
+    protocol: TCP
+  selector:
+    app: "{{ .Chart.Name }}"
+    stand: "{{ .Values.stand }}"

build/helm/templates/vault.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.vault.enabled }}
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: "{{ .Chart.Name }}"
+spec:
+  refreshInterval: "60s"
+  secretStoreRef:
+    name: "{{ .Values.vault.secretStore }}" 
+    kind: SecretStore
+  target:
+    name: "{{ .Values.vault.target.name }}"
+  data:
+    {{- range .Values.vault.data }}
+    - secretKey: {{ .secretKey }}
+      remoteRef:
+        key: "{{ .remoteRef.key }}"
+        property: "{{ .remoteRef.property }}"
+    {{- end }}
+{{- end }}

build/helm/values.yaml

@@ -0,0 +1,116 @@
+#
+# Данный конфигурационный файл не является конечным исполнением
+# Параметры подставлены исключительно для тестов и могут быть измененены в соответствии с Pipeline
+#
+
+stand: test
+
+mainContainer:
+  ###### Репозиторий
+  image:
+    repository: {{ registryUrl }}/{{ registryContainerName }}
+    tag: {{ version }}
+  containerPort: {{ appPort }}
+  ###### Ресурсы
+  resources:
+    requests: 
+      cpu: "500m"  
+      memory: "1024Mi" 
+    limits:
+      cpu: "1000m"
+      memory: "1024Mi" 
+  ###### Пробы контейнера
+  readinessProbe:
+    httpGet:
+      path: /
+      port: {{ appPort }}
+      scheme: HTTP
+  livenessProbe:
+    tcpSocket:
+      port: {{ appPort }}
+  ###### Подключение Volumes
+  volumeMounts:
+    - name: app-logs
+      mountPath: /usr/lib/serverHome/logs/
+  # Общий блок volumes для всех контейнеров
+  volumes:
+  - name: app-logs
+
+sidecars:
+  ###### Репозиторий
+  logger:
+    enabled: true
+    image: 
+      repository: drn.tst.nubes.ru/fluent-bit
+      tag: 2.2.1
+  ###### Ресурсы
+    resources:
+      requests:
+        cpu: 100m
+        memory: 256Mi
+      limits:
+        cpu: 100m
+        memory: 256Mi
+  ###### Подключение Volumes
+    volumeMounts:
+      - name: app-logs
+        mountPath: /usr/lib/serverHome/logs/
+
+
+# Для Vault есть условие
+# Предварительно в Namespace должен быть создан SecretStore
+# 
+# Читать: https://confluence.ad.nubes.ru/pages/viewpage.action?pageId=58626585
+# (Вариант 2. Используя external-operator)
+
+vault:
+  secretStore: "{{ namespace }}"
+  enabled: true
+  serviceAccountName: "{{ namespace }}-sa"
+  target:
+    name: "svc-api-envs"
+  data:
+    - secretKey: LUCEE_EXTENSIONS 
+      remoteRef:
+        key: "deck/kubernetes/{{ stand}}/{{ appName }}"
+        property: LUCEE_EXTENSIONS 
+
+    - secretKey: ORCHESTRATOR_AUTH
+      remoteRef:
+        key: "deck/kubernetes/{{ stand}}/{{ appName }}"
+        property: ORCHESTRATOR_AUTH 
+
+    - secretKey: cfconfig_datasources_cmdb_database
+      remoteRef:
+        key: "deck/kubernetes/{{ stand}}/{{ appName }}"
+        property: cfconfig_datasources_cmdb_database 
+
+    - secretKey: cfconfig_datasources_cmdb_dbdriver
+      remoteRef:
+        key: "deck/kubernetes/{{ stand}}/{{ appName }}"
+        property: cfconfig_datasources_cmdb_dbdriver 
+
+    - secretKey: cfconfig_datasources_cmdb_host
+      remoteRef:
+        key: "deck/kubernetes/{{ stand}}/{{ appName }}"
+        property: cfconfig_datasources_cmdb_host 
+
+    - secretKey: cfconfig_datasources_cmdb_name
+      remoteRef:
+        key: "deck/kubernetes/{{ stand}}/{{ appName }}"
+        property: cfconfig_datasources_cmdb_name 
+
+    - secretKey: cfconfig_datasources_cmdb_password
+      remoteRef:
+        key: "deck/kubernetes/{{ stand}}/{{ appName }}"
+        property: cfconfig_datasources_cmdb_password 
+
+    - secretKey: cfconfig_datasources_cmdb_port
+      remoteRef:
+        key: "deck/kubernetes/{{ stand}}/{{ appName }}"
+        property: cfconfig_datasources_cmdb_port 
+
+    - secretKey: cfconfig_datasources_cmdb_username
+      remoteRef:
+        key: "deck/kubernetes/{{ stand}}/{{ appName }}"
+        property: cfconfig_datasources_cmdb_username 

build/microservice.yaml.tmpl

@@ -1,121 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app: {{ appName }}
-    stand: {{ stand }}
-  name: {{ appName }}
-  namespace: {{ namespace }}
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: {{ appName }}
-      stand: {{ stand }}
-  template:
-    metadata:
-      labels:
-        app: {{ appName }}
-        stand: {{ stand }}
-    spec:
-      containers:
-        - image: {{ registryUrl }}/{{ registryContainerName }}:{{ version }}
-          imagePullPolicy: IfNotPresent
-          name: commandbox-lucee
-          ports:
-          - containerPort: {{ appPort }}
-            protocol: TCP
-          readinessProbe:
-            failureThreshold: 3
-            initialDelaySeconds: 10
-            periodSeconds: 30
-            successThreshold: 1
-            timeoutSeconds: 2
-            httpGet:
-              path: /
-              port: {{ appPort }}
-              scheme: HTTP
-          livenessProbe:
-            failureThreshold: 3
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            successThreshold: 1
-            timeoutSeconds: 1
-            tcpSocket:
-              port: {{ appPort }}
-          resources:
-            requests:
-              cpu: 500m
-              memory: 1024Mi
-            limits:
-              cpu: 500m
-              memory: 1024Mi
-          envFrom:
-          - secretRef:
-              name: envs-secrets-{{ appName }}
-#          envFrom:
-#            - secretRef:
-#                name: deploymentSecrets
-          volumeMounts:
-            - name: app-logs
-              mountPath: /usr/lib/serverHome/logs/
-        - image: drn.tst.nubes.ru/fluent-bit:2.2.1
-          resources:
-            requests:
-              cpu: 100m
-              memory: 256Mi
-            limits:
-              cpu: 100m
-              memory: 256Mi
-          imagePullPolicy: IfNotPresent
-          name: sidecar-filebeat
-          volumeMounts:
-            - name: app-logs
-              mountPath: /usr/lib/serverHome/logs/
-            - name: fluentbit-config
-              mountPath: /fluent-bit/etc/
-      volumes:
-        - name: app-logs
-        - name: fluentbit-config
-          configMap:
-            name: {{ fbConfigMap }}
-
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ appName }}
-  namespace: {{ namespace }}
-  labels:
-    app: {{ appName }}
-spec:
-  ports:
-  - port: {{ appPort }}
-    protocol: TCP
-  selector:
-    app: {{ appName }}
-    
-# ---
-# apiVersion: networking.k8s.io/v1
-# kind: Ingress
-# metadata:
-#   name: {{ appName }}
-#   namespace: {{ namespace }}
-# spec:
-#   ingressClassName: nginx
-#   rules:
-#   - host: {{ domainName }}
-#     http:
-#       paths:
-#       - backend:
-#           service:
-#             name: {{ appName }}
-#             port:
-#               number: {{ appPort }}
-#         path: /
-#         pathType: Prefix
-#   tls:
-#   - hosts:
-#     - {{ domainName }}
-#     secretName: api-deck-tls

build/vars/build.yaml

@@ -2,18 +2,12 @@ main:
   registryUrl: "drn.tst.nubes.ru"                         # Где хранится приложение
   appName: "svc-api"                                      # наименование контейнера в K8S
   registryContainerName: "deck-{{ appName }}"             # Наименование приложения в Nexus 
+  vaultPath: "config.deck-{{ appName }}-{{ stand }}"      # Путь где лежит инфраструктурный kubeconfig для установки Helm-чарта
   namespace: "deck"                                       # Namespace приложения, где он будет развёрнут
   appPort: 8080                                           # Порт, на котором работает приложение
 
-fluentBit:
-  fbConfigMap: fluentbit-configmap-api
-
 version:
   release: 0
   major: 0
   minor: "{{ BUILD_NUMBER }}"
   version: "{{ release }}.{{ major }}.{{ minor }}"
-
-# Если понадобится делать envFromSecret
-# jsonSecrets:
-#   VUE_APP_BACKEND_HOST: "{{ domainName }}"