Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Google Analytics reporting in place of existing custom reporting (#2015)

Browse Source 
Replacing the older analytics with a new Google Analytics based mechanism.
This commit is contained in:
Sanket Sudake
2021-05-12 12:24:27 +05:30
committed by GitHub
parent 040763ebb2
commit 2a014e56b0
23 changed files with 378 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -5,6 +5,7 @@ cmd/fission-cli/fission-cli
cmd/fetcher/fetcher
cmd/builder/builder
cmd/preupgradechecks/pre-upgrade-checks
cmd/reporter/reporter
# Logs
test/logs/
Makefile
+4
View File
@@ -37,6 +37,7 @@ build:
go build -o cmd/fission-cli/fission ./cmd/fission-cli/
go build -o cmd/fetcher/fetcher ./cmd/fetcher/
go build -o cmd/fetcher/builder ./cmd/builder/
go build -o cmd/reporter/reporter ./cmd/reporter/
# install CLI binary to $PATH
install: build
@@ -47,6 +48,7 @@ image:
docker build -t fission-bundle -f cmd/fission-bundle/Dockerfile.fission-bundle .
docker build -t fetcher -f cmd/fetcher/Dockerfile.fission-fetcher .
docker build -t builder -f cmd/builder/Dockerfile.fission-builder .
docker build -t reporter -f cmd/builder/Dockerfile.reporter .
# build multi-architecture images for release.
image-multiarch:
@@ -54,9 +56,11 @@ image-multiarch:
docker buildx build --platform=$(PLATFORMS) -t $(REPO)/fetcher:$(TAG) --push -f cmd/fetcher/Dockerfile.fission-fetcher .
docker buildx build --platform=$(PLATFORMS) -t $(REPO)/builder:$(TAG) --push -f cmd/builder/Dockerfile.fission-builder .
docker buildx build --platform=$(PLATFORMS) -t $(REPO)/preupgradechecks:$(TAG) --push -f cmd/preupgradechecks/Dockerfile.fission-preupgradechecks .
docker buildx build --platform=$(PLATFORMS) -t $(REPO)/reporter:$(TAG) --push -f cmd/reporter/Dockerfile.reporter .
clean:
@rm -f cmd/fission-bundle/fission-bundle
@rm -f cmd/fission-cli/fission
@rm -f cmd/fetcher/fetcher
@rm -f cmd/fetcher/builder
@rm -f cmd/reporter/reporter
charts/fission-all/templates/nonhelm-install-job.yaml
+11 -2
View File
@@ -26,7 +26,16 @@ spec:
restartPolicy: Never
containers:
- name: post-install-job
image: "fission/alpinecurl"
{{- if .Values.imageTag }}
image: {{ .Values.postInstallReportImage }}:{{ .Values.imageTag }}
{{- else }}
image: {{ .Values.postInstallReportImage }}
{{- end }}
imagePullPolicy: {{ .Values.pullPolicy }}
command: ["sh", "-c", "/usr/bin/curl -m 5 -H \"Content-Type: application/json\" -X POST -d '{\"type\": \"yaml-post-install\", \"chartName\": \"{{ .Chart.Name }}\", \"chartVersion\": \"{{ .Chart.Version }}\"}' https://g.fission.io/analytics || true"]
command: [ "/reporter" ]
args: ["event", "-c", "fission-use", "-a", "yaml-post-install", "-l", "{{ .Chart.Name }}-{{ .Chart.Version }}"]
env:
- name: GA_TRACKING_ID
value: "{{ .Values.gaTrackingID }}"
serviceAccountName: fission-svc
{{- end }}
charts/fission-all/templates/post-install-job.yaml
+11 -2
View File
@@ -30,7 +30,16 @@ spec:
restartPolicy: Never
containers:
- name: post-install-job
image: "fission/alpinecurl"
{{- if .Values.imageTag }}
image: {{ .Values.postInstallReportImage }}:{{ .Values.imageTag }}
{{- else }}
image: {{ .Values.postInstallReportImage }}
{{- end }}
imagePullPolicy: {{ .Values.pullPolicy }}
command: ["sh", "-c", "/usr/bin/curl -m 5 -H \"Content-Type: application/json\" -X POST -d '{\"type\": \"helm-post-install\", \"chartName\": \"{{ .Chart.Name }}\", \"chartVersion\": \"{{ .Chart.Version }}\"}' https://g.fission.io/analytics || true"]
command: [ "/reporter" ]
args: ["event", "-c", "fission-use", "-a", "helm-post-install", "-l", "{{ .Chart.Name }}-{{ .Chart.Version }}"]
env:
- name: GA_TRACKING_ID
value: "{{ .Values.gaTrackingID }}"
serviceAccountName: fission-svc
{{- end }}
charts/fission-all/templates/post-upgrade-job.yaml
+11 -2
View File
@@ -30,7 +30,16 @@ spec:
restartPolicy: Never
containers:
- name: post-upgrade-job
image: "fission/alpinecurl"
{{- if .Values.imageTag }}
image: {{ .Values.postInstallReportImage }}:{{ .Values.imageTag }}
{{- else }}
image: {{ .Values.postInstallReportImage }}
{{- end }}
imagePullPolicy: {{ .Values.pullPolicy }}
command: ["sh", "-c", "/usr/bin/curl -m 5 -H \"Content-Type: application/json\" -X POST -d '{\"type\": \"helm-post-upgrade\", \"chartName\": \"{{ .Chart.Name }}\", \"chartVersion\": \"{{ .Chart.Version }}\"}' https://g.fission.io/analytics || true"]
command: [ "/reporter" ]
args: ["event", "-c", "fission-use", "-a", "helm-post-upgrade", "-l", "{{ .Chart.Name }}-{{ .Chart.Version }}"]
env:
- name: GA_TRACKING_ID
value: "{{ .Values.gaTrackingID }}"
serviceAccountName: fission-svc
{{- end }}
charts/fission-all/templates/router.yaml
-7
View File
@@ -65,13 +65,6 @@ spec:
value: {{ .Values.debugEnv | quote }}
- name: DISPLAY_ACCESS_LOG
value: {{ .Values.router.displayAccessLog | default false | quote }}
{{- if .Values.analytics }}
- name: ANALYTICS_URL
value: "https://g.fission.io/metrics"
{{- else if .Values.analyticsNonHelmInstall }}
- name: ANALYTICS_URL
value: "https://g.fission.io/metrics"
{{- end }}
readinessProbe:
httpGet:
path: "/router-healthz"
charts/fission-all/values.yaml
+6
View File
@@ -270,6 +270,9 @@ analytics: true
## Internally used for generating an analytics job for non-helm installs
analyticsNonHelmInstall: false
## Google Analytics Tracking ID
gaTrackingID: UA-196546703-1
## Enable Heapster only in clusters where heapster does not exist already
heapster: false
@@ -289,6 +292,9 @@ pruneInterval: 60
## Fission pre-install/pre-upgrade checks live in this image
preUpgradeChecksImage: fission/pre-upgrade-checks
## Fission ppost-install/post-upgrade reporting live in this image
postInstallReportImage: fission/reporter
## if there are any pod specialization errors when a function is triggered and this flag is set to true, the error
## summary is returned as part of http response
debugEnv: false
charts/fission-core/templates/nonhelm-install-job.yaml
+12 -2
View File
@@ -26,6 +26,16 @@ spec:
restartPolicy: Never
containers:
- name: post-install-job
image: "fission/alpinecurl"
command: ["sh", "-c", "/usr/bin/curl -m 5 -H \"Content-Type: application/json\" -X POST -d '{\"type\": \"yaml-post-install\", \"chartName\": \"{{ .Chart.Name }}\", \"chartVersion\": \"{{ .Chart.Version }}\"}' https://g.fission.io/analytics || true"]
{{- if .Values.imageTag }}
image: {{ .Values.postInstallReportImage }}:{{ .Values.imageTag }}
{{- else }}
image: {{ .Values.postInstallReportImage }}
{{- end }}
imagePullPolicy: {{ .Values.pullPolicy }}
command: [ "/reporter" ]
args: ["event", "-c", "fission-use", "-a", "yaml-post-install", "-l", "{{ .Chart.Name }}-{{ .Chart.Version }}"]
env:
- name: GA_TRACKING_ID
value: "{{ .Values.gaTrackingID }}"
serviceAccountName: fission-svc
{{- end }}
charts/fission-core/templates/post-install-job.yaml
+12 -2
View File
@@ -30,6 +30,16 @@ spec:
restartPolicy: Never
containers:
- name: post-install-job
image: "fission/alpinecurl"
command: ["sh", "-c", "/usr/bin/curl -m 5 -H \"Content-Type: application/json\" -X POST -d '{\"type\": \"helm-post-install\", \"chartName\": \"{{ .Chart.Name }}\", \"chartVersion\": \"{{ .Chart.Version }}\"}' https://g.fission.io/analytics || true"]
{{- if .Values.imageTag }}
image: {{ .Values.postInstallReportImage }}:{{ .Values.imageTag }}
{{- else }}
image: {{ .Values.postInstallReportImage }}
{{- end }}
imagePullPolicy: {{ .Values.pullPolicy }}
command: [ "/reporter" ]
args: ["event", "-c", "fission-use", "-a", "helm-post-install", "-l", "{{ .Chart.Name }}-{{ .Chart.Version }}"]
env:
- name: GA_TRACKING_ID
value: "{{ .Values.gaTrackingID }}"
serviceAccountName: fission-svc
{{- end }}
charts/fission-core/templates/post-upgrade-job.yaml
+12 -2
View File
@@ -30,6 +30,16 @@ spec:
restartPolicy: Never
containers:
- name: post-upgrade-job
image: "fission/alpinecurl"
command: ["sh", "-c", "/usr/bin/curl -m 5 -H \"Content-Type: application/json\" -X POST -d '{\"type\": \"helm-post-upgrade\", \"chartName\": \"{{ .Chart.Name }}\", \"chartVersion\": \"{{ .Chart.Version }}\"}' https://g.fission.io/analytics || true"]
{{- if .Values.imageTag }}
image: {{ .Values.postInstallReportImage }}:{{ .Values.imageTag }}
{{- else }}
image: {{ .Values.postInstallReportImage }}
{{- end }}
imagePullPolicy: {{ .Values.pullPolicy }}
command: [ "/reporter" ]
args: ["event", "-c", "fission-use", "-a", "helm-post-upgrade", "-l", "{{ .Chart.Name }}-{{ .Chart.Version }}"]
env:
- name: GA_TRACKING_ID
value: "{{ .Values.gaTrackingID }}"
serviceAccountName: fission-svc
{{- end }}
charts/fission-core/templates/router.yaml
-7
View File
@@ -65,13 +65,6 @@ spec:
value: {{ .Values.debugEnv | quote }}
- name: DISPLAY_ACCESS_LOG
value: {{ .Values.router.displayAccessLog | default false | quote }}
{{- if .Values.analytics }}
- name: ANALYTICS_URL
value: "https://g.fission.io/metrics"
{{- else if .Values.analyticsNonHelmInstall }}
- name: ANALYTICS_URL
value: "https://g.fission.io/metrics"
{{- end }}
readinessProbe:
httpGet:
path: "/router-healthz"
charts/fission-core/values.yaml
+6
View File
@@ -170,6 +170,9 @@ analytics: true
## Internally used for generating an analytics job for non-helm installs
analyticsNonHelmInstall: false
## Google Analytics Tracking ID
gaTrackingID: UA-196546703-1
## Archive pruner is a garbage collector for archives on the fission storage service.
## This interval configures the frequency at which it runs inside the storagesvc pod.
## The value is in minutes.
@@ -178,6 +181,9 @@ pruneInterval: 60
## Fission pre-install/pre-upgrade checks live in this image
preUpgradeChecksImage: fission/pre-upgrade-checks
## Fission ppost-install/post-upgrade reporting live in this image
postInstallReportImage: fission/reporter
## if there are any pod specialization errors when a function is triggered and this flag is set to true, the error
## summary is returned as part of http response
debugEnv: false
cmd/reporter/Dockerfile.reporter
+43
View File
@@ -0,0 +1,43 @@
FROM golang:1.14-alpine as godep
RUN apk add bash ca-certificates git gcc g++ libc-dev
ARG GOPKG=github.com/fission/fission
ENV GO111MODULE=on
WORKDIR /go/src/${GOPKG}
# To reuse build cache, copy go.mod & go.sum and download dependencies first.
COPY go.* ./
RUN go mod download
FROM godep as builder
ARG GOPKG
WORKDIR /go/src/${GOPKG}
# Copy whole fission directory to work dir
COPY ./ ./
WORKDIR /go/src/${GOPKG}/cmd/reporter
ARG GITCOMMIT=unknown
# E.g. GITCOMMIT=$(git rev-parse HEAD)
ARG BUILDVERSION=unknown
# E.g. BUILDVERSION=$(git rev-parse HEAD)
ARG BUILDDATE=unknown
# E.g. BUILDDATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
RUN CGO_ENABLED=0 go build \
-o /go/bin/reporter \
-gcflags=-trimpath=$GOPATH \
-asmflags=-trimpath=$GOPATH \
-ldflags "-X github.com/fission/fission/pkg/info.GitCommit=${GITCOMMIT} -X github.com/fission/fission/pkg/info.BuildDate=${BUILDDATE} -X github.com/fission/fission/pkg/info.Version=${BUILDVERSION}"
FROM alpine:3.13 as base
RUN apk add --update ca-certificates
COPY --from=builder /go/bin/reporter /
ENTRYPOINT ["/reporter"]
cmd/reporter/app/app.go
+33
View File
@@ -0,0 +1,33 @@
/*
Copyright 2021 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"github.com/spf13/cobra"
)
func App() *cobra.Command {
cobra.EnableCommandSorting = false
rootCmd := &cobra.Command{
Use: "reporter",
Short: "Report fission events to analytics service",
ValidArgs: []string{"event"},
Args: cobra.OnlyValidArgs,
}
rootCmd.AddCommand(EventCommand())
return rootCmd
}
cmd/reporter/app/cmd_event.go
+67
View File
@@ -0,0 +1,67 @@
/*
Copyright 2021 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"github.com/fission/fission/pkg/tracker"
"github.com/spf13/cobra"
)
func eventCommandHandler(cmd *cobra.Command, args []string) error {
var err error
flags := cmd.PersistentFlags()
event := tracker.Event{}
event.Category, err = flags.GetString("category")
if err != nil {
return err
}
event.Action, err = flags.GetString("action")
if err != nil {
return err
}
event.Label, err = flags.GetString("label")
if err != nil {
return err
}
event.Value, err = flags.GetString("value")
if err != nil {
return err
}
return tracker.Tracker.SendEvent(event)
}
func EventCommand() *cobra.Command {
eventCmd := &cobra.Command{
Use: "event",
Short: "Report event to analytics",
RunE: eventCommandHandler,
Args: cobra.NoArgs,
}
persistentFlags := eventCmd.PersistentFlags()
persistentFlags.StringP("category", "c", "", "event category")
persistentFlags.StringP("action", "a", "", "event action")
persistentFlags.StringP("label", "l", "", "event label")
persistentFlags.StringP("value", "v", "", "event value")
eventCmd.MarkPersistentFlagRequired("category")
eventCmd.MarkPersistentFlagRequired("action")
return eventCmd
}
cmd/reporter/main.go
+25
View File
@@ -0,0 +1,25 @@
/*
Copyright 2021 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"github.com/fission/fission/cmd/reporter/app"
)
func main() {
app.App().Execute()
}
hack/release-build.sh
+14
View File
@@ -85,6 +85,19 @@ build_builder_image() {
docker tag $tag fission/builder:latest
}
# Build reporter image
build_reporter_image() {
local version=$1
local date=$2
local gitcommit=$3
local tag=fission/reporter:$version
docker build -t $tag -f $DIR/cmd/reporter/Dockerfile.reporter \
--build-arg GITCOMMIT=$gitcommit --build-arg BUILDDATE=$date --build-arg BUILDVERSION=$version $DIR
docker tag $tag fission/reporter:latest
}
# Build pre-upgrade-checks image
build_pre_upgrade_checks_image() {
local version=$1
@@ -182,6 +195,7 @@ build_all() {
build_builder_image $version $date $gitcommit
build_all_cli $version $date $gitcommit
build_pre_upgrade_checks_image $version $date $gitcommit
build_reporter_image $version $date $gitcommit
remove_generated_swagger_doc
}
pkg/router/analytics.go
-69
View File
@@ -1,69 +0,0 @@
package router
import (
"bytes"
"encoding/json"
"net/http"
"os"
"sync/atomic"
"time"
"github.com/dchest/uniuri"
)
type (
// Analytics helps exporting usage metrics
Analytics struct {
id string
url string
}
// AnalyticsData is the data exported as usage metrics
AnalyticsData struct {
ID string `json:"ID"`
FunctionCallCount uint64 `json:"FunctionCallCount"`
}
)
// MakeAnalytics returns a new instance of Analytics if url or
// 'ANALYTICS_URL' environment variable is set; nil otherwise
func MakeAnalytics(url string) *Analytics {
if len(url) == 0 {
url = os.Getenv("ANALYTICS_URL")
if len(url) == 0 {
return nil
}
}
a := &Analytics{
url: url,
id: uniuri.NewLen(8),
}
go a.run()
return a
}
func (a *Analytics) gatherData() *AnalyticsData {
return &AnalyticsData{
FunctionCallCount: atomic.LoadUint64(&globalFunctionCallCount),
}
}
func (a *Analytics) run() {
ticker := time.NewTicker(24 * time.Hour)
for range ticker.C {
msg := a.gatherData()
msg.ID = a.id
msgbytes, err := json.Marshal(*msg)
if err != nil {
continue
}
resp, _ := http.Post(a.url, "application/json", bytes.NewReader(msgbytes))
if resp != nil {
// close response body to prevent resources leak
resp.Body.Close()
}
}
}
pkg/router/router.go
-2
View File
@@ -119,8 +119,6 @@ func serveMetric(logger *zap.Logger) {
// Start starts a router
func Start(logger *zap.Logger, port int, executorURL string) {
_ = MakeAnalytics("")
fmap := makeFunctionServiceMap(logger, time.Minute)
fissionClient, kubeClient, _, _, err := crd.MakeFissionClient()
pkg/tracker/tracker.go
+101
View File
@@ -0,0 +1,101 @@
/*
Copyright 2021 The Fission Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracker
import (
"bytes"
"context"
"errors"
"net/http"
"net/url"
"os"
"time"
uuid "github.com/satori/go.uuid"
)
const HTTP_TIMEOUT = 5 * time.Second
const GA_API_URL = "https://www.google-analytics.com/collect"
const GA_TRACKING_ID = "GA_TRACKING_ID"
var Tracker *tracker
type (
tracker struct {
gaPropertyID string
cid string
}
Event struct {
Category string
Action string
Label string
Value string
}
)
func init() {
Tracker = &tracker{gaPropertyID: os.Getenv(GA_TRACKING_ID), cid: uuid.NewV4().String()}
}
func (t *tracker) SendEvent(e Event) error {
if t.gaPropertyID == "" {
return errors.New("tracker.SendEvent: GA_TRACKING_ID env not set")
}
if e.Action == "" || e.Category == "" {
return errors.New("tracker.SendEvent: category and action are required")
}
v := url.Values{
"v": {"1"},
"tid": {t.gaPropertyID},
"cid": {t.cid},
"t": {"event"},
"ec": {e.Category},
"ea": {e.Action},
}
if e.Label != "" {
v.Add("el", e.Label)
}
if e.Value != "" {
v.Add("ev", e.Value)
}
buf := bytes.NewBufferString(v.Encode())
req, err := http.NewRequest("POST", GA_API_URL, buf)
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
req.Header.Add("User-Agent", "ga-tracker/1.0")
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(req.Context(), HTTP_TIMEOUT)
defer cancel()
req = req.WithContext(ctx)
client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return errors.New("tracker.SendEvent: analytics response status not ok")
}
defer resp.Body.Close()
return err
}
skaffold.yaml
+5
View File
@@ -24,6 +24,9 @@ build:
- image: preupgradechecks
docker:
dockerfile: cmd/preupgradechecks/Dockerfile.fission-preupgradechecks
- image: reporter
docker:
dockerfile: cmd/reporter/Dockerfile.reporter
deploy:
helm:
@@ -41,6 +44,7 @@ deploy:
artifactOverrides:
image: "fission"
preUpgradeChecksImage: "preupgradechecks"
postInstallReportImage: "reporter"
fetcher.image: "fetcher"
setValues:
namespace: fission
@@ -51,6 +55,7 @@ deploy:
imageTag: ""
prometheus.enabled: false
debugEnv: false
analytics: false
wait: true
recreatePods: false
packaged: null
test/tests/mqtrigger/nats/test_mqtrigger.sh
+2
View File
@@ -60,6 +60,8 @@ go run $DIR/stan-pub/main.go -s $FISSION_NATS_STREAMING_URL -c $clusterID -id $p
#
log "Waiting for response"
response=$(timeout 5s go run $DIR/stan-sub/main.go --last -s $FISSION_NATS_STREAMING_URL -c $clusterID -id $subClientID $resptopic 2>&1 || true)
log "Output from subscriber"
echo "$response"
echo "$response" | grep "$expectedRespOutput"
log "Deleting message queue trigger"
test/tests/mqtrigger/nats/test_mqtrigger_error.sh
+2
View File
@@ -64,6 +64,8 @@ go run $DIR/stan-pub/main.go -s $FISSION_NATS_STREAMING_URL -c $clusterID -id $p
#
log "Waiting for response"
response=$(timeout 5s go run $DIR/stan-sub/main.go --last -s $FISSION_NATS_STREAMING_URL -c $clusterID -id $subClientID $errortopic 2>&1 || true)
log "Output from subscriber"
echo "$response"
echo "$response" | grep "$expectedRespOutput"
log "Deleting message queue trigger"