Nail/fission-src
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use zap for logging (#1112)

Browse Source 
Use zap for logging
This commit is contained in:
Jon Carl
2019-03-14 21:11:15 +05:30
committed by Vishal
parent c717f182b6
commit 0fc864f230
98 changed files with 2031 additions and 1223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
storagesvc/archivePruner.go
+29 -20
View File
@@ -19,13 +19,14 @@ package storagesvc
import (
"time"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/fission/fission/crd"
)
type ArchivePruner struct {
logger *zap.Logger
crdClient *crd.FissionClient
archiveChan chan (string)
stowClient *StowClient
@@ -34,13 +35,14 @@ type ArchivePruner struct {
const defaultPruneInterval int = 60 // in minutes
func MakeArchivePruner(stowClient *StowClient, pruneInterval time.Duration) (*ArchivePruner, error) {
func MakeArchivePruner(logger *zap.Logger, stowClient *StowClient, pruneInterval time.Duration) (*ArchivePruner, error) {
crdClient, _, _, err := crd.MakeFissionClient()
if err != nil {
return nil, err
}
return &ArchivePruner{
logger: logger.Named("archive_pruner"),
crdClient: crdClient,
archiveChan: make(chan string),
stowClient: stowClient,
@@ -50,15 +52,18 @@ func MakeArchivePruner(stowClient *StowClient, pruneInterval time.Duration) (*Ar
// pruneArchives listens to archiveChannel for archive ids that need to be deleted
func (pruner *ArchivePruner) pruneArchives() {
log.Info("listening to archiveChannel to prune archives")
pruner.logger.Info("listening to archiveChannel to prune archives")
for {
select {
case archiveID := <-pruner.archiveChan:
log.WithField("archive ID", archiveID).Info("sending delete request")
pruner.logger.Info("sending delete request for archive",
zap.String("archive_id", archiveID))
if err := pruner.stowClient.removeFileByID(archiveID); err != nil {
// logging the error and continuing with other deletions.
// hopefully this archive will be deleted in the next iteration.
log.WithField("archiveID", archiveID).WithError(err).Error("Ignoring error while deleting archive")
pruner.logger.Error("ignoring error while deleting archive",
zap.Error(err),
zap.String("archive_id", archiveID))
}
}
}
@@ -72,16 +77,16 @@ func (pruner *ArchivePruner) insertArchive(archiveID string) {
// A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes
// and not the archives that are referenced by them, leaving the archives as orphans.
// getOrphanArchives reaps the orphaned archives.
func (pruner *ArchivePruner) getOrphanArchives() error {
log.Info("getting orphan archives")
func (pruner *ArchivePruner) getOrphanArchives() {
pruner.logger.Info("getting orphan archives")
archivesRefByPkgs := make([]string, 0)
var archiveID string
// get all pkgs from kubernetes
pkgList, err := pruner.crdClient.Packages(metav1.NamespaceAll).List(metav1.ListOptions{})
if err != nil {
log.WithError(err).Error("Error getting package list from kubernetes")
return err
pruner.logger.Error("error getting package list from kubernetes", zap.Error(err))
return
}
// extract archives referenced by these pkgs
@@ -89,44 +94,48 @@ func (pruner *ArchivePruner) getOrphanArchives() error {
if pkg.Spec.Deployment.URL != "" {
archiveID, err = getQueryParamValue(pkg.Spec.Deployment.URL, "id")
if err != nil {
log.WithError(err).Error("Error extracting value of archiveID from url")
return err
pruner.logger.Error("error extracting value of archiveID from deployment url",
zap.Error(err),
zap.String("url", pkg.Spec.Deployment.URL))
return
}
archivesRefByPkgs = append(archivesRefByPkgs, archiveID)
}
if pkg.Spec.Source.URL != "" {
archiveID, err = getQueryParamValue(pkg.Spec.Source.URL, "id")
if err != nil {
log.WithError(err).Error("Error extracting value of archiveID from url")
return err
pruner.logger.Error("error extracting value of archiveID from source url",
zap.Error(err),
zap.String("url", pkg.Spec.Source.URL))
return
}
archivesRefByPkgs = append(archivesRefByPkgs, archiveID)
}
}
log.WithField("list", "archives referenced by packages").Debugf("%s", archivesRefByPkgs)
pruner.logger.Debug("archives referenced by packagese", zap.Strings("archives", archivesRefByPkgs))
// get all archives on storage
// out of them, there may be some just created but not referenced by packages yet.
// need to filter them out.
archivesInStorage, err := pruner.stowClient.getItemIDsWithFilter(filterItemCreatedAMinuteAgo, time.Now())
archivesInStorage, err := pruner.stowClient.getItemIDsWithFilter(pruner.stowClient.filterItemCreatedAMinuteAgo, time.Now())
if err != nil {
log.WithError(err).Error("Error getting items from storage")
return err
pruner.logger.Error("error getting items from storage", zap.Error(err))
return
}
log.WithField("list", "archives in storage").Debugf("%s", archivesInStorage)
pruner.logger.Debug("archives in storage", zap.Strings("archives", archivesInStorage))
// difference of the two lists gives us the list of orphan archives. This is just a brute force approach.
// need to do something more optimal at scale.
orphanedArchives := getDifferenceOfLists(archivesInStorage, archivesRefByPkgs)
log.WithField("list", "orphan archives").Debugf("%s", orphanedArchives)
pruner.logger.Debug("orphan archives", zap.Strings("archives", orphanedArchives))
// send each orphan archive away for deletion
for _, archiveID = range orphanedArchives {
pruner.insertArchive(archiveID)
}
return nil
return
}
// Start starts a go routine that listens to a channel for archive IDs that need to deleted.
storagesvc/client/storagesvc_test.go
+5 -1
View File
@@ -27,6 +27,7 @@ import (
"time"
"github.com/dchest/uniuri"
"go.uber.org/zap"
"github.com/fission/fission/storagesvc"
)
@@ -52,9 +53,12 @@ func TestStorageService(t *testing.T) {
port := 8080
enableArchivePruner := false
logger, err := zap.NewDevelopment()
panicIf(err)
log.Println("starting storage svc")
_ = storagesvc.RunStorageService(
storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner)
logger, storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner)
time.Sleep(time.Second)
client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))
storagesvc/storagesvc.go
+30 -20
View File
@@ -25,15 +25,17 @@ import (
"strconv"
"time"
"github.com/fission/fission"
"github.com/gorilla/mux"
_ "github.com/graymeta/stow/local"
log "github.com/sirupsen/logrus"
"go.opencensus.io/plugin/ochttp"
"go.uber.org/zap"
"github.com/fission/fission"
)
type (
StorageService struct {
logger *zap.Logger
storageClient *StowClient
port int
}
@@ -61,27 +63,34 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request)
fileSizeS, ok := r.Header["X-File-Size"]
if !ok {
log.Error("Missing X-File-Size")
ss.logger.Error("upload is missing the 'X-File-Size' header",
zap.String("filename", handler.Filename))
http.Error(w, "missing X-File-Size header", http.StatusBadRequest)
return
}
fileSize, err := strconv.Atoi(fileSizeS[0])
if err != nil {
log.WithError(err).Errorf("Error parsing x-file-size: '%v'", fileSizeS)
ss.logger.Error("error parsing 'X-File-Size' header",
zap.Error(err),
zap.Strings("header", fileSizeS),
zap.String("filename", handler.Filename))
http.Error(w, "missing or bad X-File-Size header", http.StatusBadRequest)
return
}
// TODO: allow headers to add more metadata (e.g. environment
// and function metadata)
log.Infof("Handling upload for %v", handler.Filename)
ss.logger.Info("handling upload",
zap.String("filename", handler.Filename))
//fileMetadata := make(map[string]interface{})
//fileMetadata["filename"] = handler.Filename
id, err := ss.storageClient.putFile(file, int64(fileSize))
if err != nil {
log.WithError(err).Error("Error saving uploaded file")
ss.logger.Error("error saving uploaded file",
zap.Error(err),
zap.String("filename", handler.Filename))
http.Error(w, "Error saving uploaded file", http.StatusInternalServerError)
return
}
@@ -92,6 +101,9 @@ func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request)
}
resp, err := json.Marshal(ur)
if err != nil {
ss.logger.Error("error marshaling uploaded file response",
zap.Error(err),
zap.String("filename", handler.Filename))
http.Error(w, "Error marshaling response", http.StatusInternalServerError)
return
}
@@ -136,7 +148,7 @@ func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request
// stream it to response
err = ss.storageClient.copyFileToStream(fileId, w)
if err != nil {
log.WithError(err).Errorf("Error getting item id '%v'", fileId)
ss.logger.Error("error getting file from storage client", zap.Error(err), zap.String("file_id", fileId))
if err == ErrNotFound {
http.Error(w, "Error retrieving item: not found", http.StatusNotFound)
} else if err == ErrRetrievingItem {
@@ -154,8 +166,9 @@ func (ss *StorageService) healthHandler(w http.ResponseWriter, r *http.Request)
w.WriteHeader(http.StatusOK)
}
func MakeStorageService(storageClient *StowClient, port int) *StorageService {
func MakeStorageService(logger *zap.Logger, storageClient *StowClient, port int) *StorageService {
return &StorageService{
logger: logger.Named("storage_service"),
storageClient: storageClient,
port: port,
}
@@ -170,30 +183,27 @@ func (ss *StorageService) Start(port int) {
address := fmt.Sprintf(":%v", port)
r.Use(fission.LoggingMiddleware)
r.Use(fission.LoggingMiddleware(ss.logger))
err := http.ListenAndServe(address, &ochttp.Handler{
Handler: r,
// Propagation: &b3.HTTPFormat{},
})
log.Fatal(err)
ss.logger.Fatal("done listening", zap.Error(err))
}
func RunStorageService(storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService {
func RunStorageService(logger *zap.Logger, storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService {
// setup a signal handler for SIGTERM
fission.SetupStackTraceHandler()
// initialize logger
log.SetLevel(log.InfoLevel)
// create a storage client
storageClient, err := MakeStowClient(storageType, storagePath, containerName)
storageClient, err := MakeStowClient(logger, storageType, storagePath, containerName)
if err != nil {
log.Fatalf("Error creating stowClient: %v", err)
logger.Fatal("error creating stowClient", zap.Error(err))
}
// create http handlers
storageService := MakeStorageService(storageClient, port)
storageService := MakeStorageService(logger, storageClient, port)
go storageService.Start(port)
// enablePruner prevents storagesvc unit test from needing to talk to kubernetes
@@ -203,13 +213,13 @@ func RunStorageService(storageType StorageType, storagePath string, containerNam
if err != nil {
pruneInterval = defaultPruneInterval
}
pruner, err := MakeArchivePruner(storageClient, time.Duration(pruneInterval))
pruner, err := MakeArchivePruner(logger, storageClient, time.Duration(pruneInterval))
if err != nil {
log.Fatalf("Error creating archivePruner: %v", err)
logger.Fatal("error creating archivePruner", zap.Error(err))
}
go pruner.Start()
}
log.Info("Storage service started")
logger.Info("storage service started")
return storageService
}
storagesvc/stowClient.go
+16 -11
View File
@@ -17,7 +17,6 @@ limitations under the License.
package storagesvc
import (
"errors"
"io"
"mime/multipart"
"os"
@@ -25,8 +24,9 @@ import (
"github.com/graymeta/stow"
_ "github.com/graymeta/stow/local"
"github.com/pkg/errors"
"github.com/satori/go.uuid"
log "github.com/sirupsen/logrus"
"go.uber.org/zap"
)
type (
@@ -40,6 +40,7 @@ type (
}
StowClient struct {
logger *zap.Logger
config *storageConfig
location stow.Location
container stow.Container
@@ -59,7 +60,7 @@ var (
ErrWritingFileIntoResponse = errors.New("unable to copy item into http response")
)
func MakeStowClient(storageType StorageType, storagePath string, containerName string) (*StowClient, error) {
func MakeStowClient(logger *zap.Logger, storageType StorageType, storagePath string, containerName string) (*StowClient, error) {
if storageType != StorageTypeLocal {
return nil, errors.New("Storage types other than 'local' are not implemented")
}
@@ -71,13 +72,13 @@ func MakeStowClient(storageType StorageType, storagePath string, containerName s
}
stowClient := &StowClient{
logger: logger.Named("stow_client"),
config: config,
}
cfg := stow.ConfigMap{"path": config.localPath}
loc, err := stow.Dial("local", cfg)
if err != nil {
log.WithError(err).Error("Error initializing storage")
return nil, err
}
stowClient.location = loc
@@ -99,7 +100,6 @@ func MakeStowClient(storageType StorageType, storagePath string, containerName s
}
}
if err != nil {
log.WithError(err).Error("Error initializing storage")
return nil, err
}
stowClient.container = con
@@ -116,11 +116,13 @@ func (client *StowClient) putFile(file multipart.File, fileSize int64) (string,
// save the file to the storage backend
item, err := client.container.Put(uploadName, file, int64(fileSize), nil)
if err != nil {
log.WithError(err).Errorf("Error writing file: %s on storage", uploadName)
client.logger.Error("error writing file on storage",
zap.Error(err),
zap.String("file", uploadName))
return "", ErrWritingFile
}
log.Debugf("Successfully wrote file:%s on storage", uploadName)
client.logger.Debug("successfully wrote file on storage", zap.String("file", uploadName))
return item.ID(), nil
}
@@ -146,7 +148,7 @@ func (client *StowClient) copyFileToStream(fileId string, w io.Writer) error {
return ErrWritingFileIntoResponse
}
log.Debugf("successfully wrote file: %s into httpresponse", fileId)
client.logger.Debug("successfully wrote file into httpresponse", zap.String("file", fileId))
return nil
}
@@ -169,7 +171,7 @@ func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncPara
for {
items, cursor, err = client.container.Items(stow.NoPrefix, cursor, PaginationSize)
if err != nil {
log.WithError(err).Error("Error getting items from container")
errors.Wrap(err, "error getting items from container")
return nil, err
}
@@ -191,10 +193,13 @@ func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncPara
// filterItemCreatedAMinuteAgo is one type of filter function that filters out items created less than a minute ago.
// More filter functions can be written if needed, as long as they are of type filter
func filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool {
func (client StowClient) filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool {
itemLastModTime, _ := item.LastMod()
if currentTime.(time.Time).Sub(itemLastModTime) < 1*time.Minute {
log.Debugf("item: %s created less than a minute ago: %v", item.ID(), itemLastModTime)
client.logger.Debug("item created less than a minute ago",
zap.String("item", item.ID()),
zap.Time("last_modified_time", itemLastModTime))
return true
}
return false
storagesvc/util.go
+3 -3
View File
@@ -17,15 +17,15 @@ limitations under the License.
package storagesvc
import (
"log"
"net/url"
"github.com/pkg/errors"
)
func getQueryParamValue(urlString string, queryParam string) (string, error) {
url, err := url.Parse(urlString)
if err != nil {
log.Printf("Error parsing URL string: %s into URL", urlString)
return "", err
return "", errors.Wrapf(err, "error parsing URL string %q into URL", urlString)
}
return url.Query().Get(queryParam), nil
}