Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prometheus metrics improvements (#2398)

Browse Source 
- Enabled metrics in storagesvc, buildermgr and controller.
- Added a middleware in storagesvc, router, executor and controller to monitor total number of http requests, each request's duration and number of requests that are currently being served. These requests can be filtered on their path, method or statuscode.
- Removed functionCallDuration and functionCallResponseSize metrics from router.
- Removed funcAliveSummary, funcIsAlive, funcReapTime and idleTime metrics.
- Replaced function calls for collecting metrics to direct metric calls.

Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Ankit Chawla
2022-04-13 21:49:46 +05:30
committed by GitHub
co-authored by Sanket Sudake
parent 231de707dc
commit b638a6d047
26 changed files with 359 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storagesvc/client/storagesvc_test.go
+25 -27
View File
@@ -76,27 +76,6 @@ func runMinioDockerContainer(pool *dockertest.Pool) *dockertest.Resource {
return resource
}
func startS3StorageService(ctx context.Context, endpoint, bucketName, subDir string) {
// testID := uniuri.NewLen(8)
port := 8081
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, err := config.Build()
panicIf(err)
log.Println("starting storage svc")
os.Setenv("STORAGE_S3_ENDPOINT", endpoint)
os.Setenv("STORAGE_S3_BUCKET_NAME", bucketName)
os.Setenv("STORAGE_S3_SUB_DIR", subDir)
os.Setenv("STORAGE_S3_ACCESS_KEY_ID", minioAccessKeyID)
os.Setenv("STORAGE_S3_SECRET_ACCESS_KEY", minioSecretAccessKey)
os.Setenv("STORAGE_S3_REGION", minioRegion)
storage := storagesvc.NewS3Storage()
_ = storagesvc.Start(ctx, logger, storage, port, true)
}
func TestS3StorageService(t *testing.T) {
fmt.Println("Test S3 Storage service")
var minioClient *minio.Client
@@ -135,7 +114,26 @@ func TestS3StorageService(t *testing.T) {
// Start storagesvc
bucketName := "test-s3-service"
subDir := "x/y/z"
startS3StorageService(context.Background(), endpoint, bucketName, subDir)
// testID := uniuri.NewLen(8)
port := 8081
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, err := config.Build()
panicIf(err)
log.Println("starting storage svc")
os.Setenv("STORAGE_S3_ENDPOINT", endpoint)
os.Setenv("STORAGE_S3_BUCKET_NAME", bucketName)
os.Setenv("STORAGE_S3_SUB_DIR", subDir)
os.Setenv("STORAGE_S3_ACCESS_KEY_ID", minioAccessKeyID)
os.Setenv("STORAGE_S3_SECRET_ACCESS_KEY", minioSecretAccessKey)
os.Setenv("STORAGE_S3_REGION", minioRegion)
storage := storagesvc.NewS3Storage()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_ = storagesvc.Start(ctx, logger, storage, port, true)
time.Sleep(time.Second)
client := MakeClient(fmt.Sprintf("http://localhost:%v/", 8081))
@@ -146,7 +144,6 @@ func TestS3StorageService(t *testing.T) {
// store it
metadata := make(map[string]string)
ctx := context.Background()
fileID, err := client.Upload(ctx, tmpfile.Name(), &metadata)
panicIf(err)
@@ -195,12 +192,11 @@ func TestS3StorageService(t *testing.T) {
if err == nil {
log.Panic("Download succeeded but file isn't supposed to exist")
}
}
func TestLocalStorageService(t *testing.T) {
testID := uniuri.NewLen(8)
port := 8080
port := 8082
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
@@ -211,7 +207,10 @@ func TestLocalStorageService(t *testing.T) {
localPath := fmt.Sprintf("/tmp/%v", testID)
_ = os.Mkdir(localPath, os.ModePerm)
storage := storagesvc.NewLocalStorage(localPath)
_ = storagesvc.Start(context.Background(), logger, storage, port, true)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
os.Setenv("METRICS_ADDR", ":8083")
_ = storagesvc.Start(ctx, logger, storage, port, true)
time.Sleep(time.Second)
client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))
@@ -222,7 +221,6 @@ func TestLocalStorageService(t *testing.T) {
// store it
metadata := make(map[string]string)
ctx := context.Background()
fileID, err := client.Upload(ctx, tmpfile.Name(), &metadata)
panicIf(err)
pkg/storagesvc/metrics.go
+24
View File
@@ -0,0 +1,24 @@
package storagesvc
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
functionLabels = []string{}
totalArchives = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "fission_archives_total",
Help: "Number of archives stored",
},
functionLabels,
)
totalMemoryUsage = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "fission_archive_memory_bytes",
Help: "Amount of memory consumed by archives",
},
functionLabels,
)
)
pkg/storagesvc/storagesvc.go
+20 -4
View File
@@ -31,6 +31,7 @@ import (
"go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
"github.com/fission/fission/pkg/utils/metrics"
"github.com/fission/fission/pkg/utils/otel"
)
@@ -115,6 +116,8 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request)
return
}
totalMemoryUsage.WithLabelValues().Add(float64(fileSize))
// respond with an ID that can be used to retrieve the file
ur := &UploadResponse{
ID: id,
@@ -135,6 +138,8 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request)
zap.String("filename", handler.Filename),
)
}
totalArchives.WithLabelValues().Inc()
}
func (ss *StorageService) getIdFromRequest(r *http.Request) (string, error) {
@@ -154,12 +159,20 @@ func (ss *StorageService) deleteHandler(w http.ResponseWriter, r *http.Request)
return
}
filesize, err := ss.storageClient.getFileSize(fileId)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
}
err = ss.storageClient.removeFileByID(fileId)
if err != nil {
msg := fmt.Sprintf("Error deleting item: %v", err)
http.Error(w, msg, http.StatusInternalServerError)
return
}
totalArchives.WithLabelValues().Dec()
totalMemoryUsage.WithLabelValues().Sub(float64(filesize))
w.WriteHeader(http.StatusOK)
}
@@ -203,6 +216,7 @@ func MakeStorageService(logger *zap.Logger, storageClient *StowClient, port int)
func (ss *StorageService) Start(port int, openTracingEnabled bool) {
r := mux.NewRouter()
r.Use(metrics.HTTPMetricMiddleware())
r.HandleFunc("/v1/archive", ss.uploadHandler).Methods("POST")
r.HandleFunc("/v1/archive", ss.downloadHandler).Methods("GET")
r.HandleFunc("/v1/archive", ss.deleteHandler).Methods("DELETE")
@@ -210,14 +224,15 @@ func (ss *StorageService) Start(port int, openTracingEnabled bool) {
address := fmt.Sprintf(":%v", port)
var err error
var handler http.Handler
if openTracingEnabled {
err = http.ListenAndServe(address, &ochttp.Handler{
handler = &ochttp.Handler{
Handler: r,
})
}
} else {
err = http.ListenAndServe(address, otel.GetHandlerWithOTEL(r, "fission-storagesvc", otel.UrlsToIgnore("/healthz")))
handler = otel.GetHandlerWithOTEL(r, "fission-storagesvc", otel.UrlsToIgnore("/healthz"))
}
err := http.ListenAndServe(address, handler)
ss.logger.Fatal("done listening", zap.Error(err))
}
@@ -232,6 +247,7 @@ func Start(ctx context.Context, logger *zap.Logger, storage Storage, port int, o
// create http handlers
storageService := MakeStorageService(logger, storageClient, port)
go metrics.ServeMetrics(ctx, logger)
go storageService.Start(port, openTracingEnabled)
// enablePruner prevents storagesvc unit test from needing to talk to kubernetes
pkg/storagesvc/stowClient.go
+12
View File
@@ -182,6 +182,18 @@ func (client *StowClient) removeFileByID(itemID string) error {
return client.container.RemoveItem(itemID)
}
func (client *StowClient) getFileSize(itemID string) (int64, error) {
item, err := client.container.Item(itemID)
if err != nil {
if err == stow.ErrNotFound {
return 0, ErrNotFound
} else {
return 0, ErrRetrievingItem
}
}
return item.Size()
}
// filter defines an interface to filter out items from a set of items
type filter func(stow.Item, interface{}) bool