Move packages to proejct/pkg to follow go project folder structure convention (#1190)

2019-05-31 16:28:55 +08:00
parent 1c5fd92ad6
commit a0e9a39511
196 changed files with 1672 additions and 1716 deletions
@@ -0,0 +1,225 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storagesvc
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/gorilla/mux"
+	_ "github.com/graymeta/stow/local"
+	"go.opencensus.io/plugin/ochttp"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/utils"
+)
+
+type (
+	StorageService struct {
+		logger        *zap.Logger
+		storageClient *StowClient
+		port          int
+	}
+
+	UploadResponse struct {
+		ID string `json:"id"`
+	}
+)
+
+// Handle multipart file uploads.
+func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request) {
+	// handle upload
+	r.ParseMultipartForm(0)
+	file, handler, err := r.FormFile("uploadfile")
+	if err != nil {
+		http.Error(w, "missing upload file", http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+
+	// stow wants the file size, but that's different from the
+	// content length, the content length being the size of the
+	// encoded file in the HTTP request. So we require an
+	// "X-File-Size" header in bytes.
+
+	fileSizeS, ok := r.Header["X-File-Size"]
+	if !ok {
+		ss.logger.Error("upload is missing the 'X-File-Size' header",
+			zap.String("filename", handler.Filename))
+		http.Error(w, "missing X-File-Size header", http.StatusBadRequest)
+		return
+	}
+
+	fileSize, err := strconv.Atoi(fileSizeS[0])
+	if err != nil {
+		ss.logger.Error("error parsing 'X-File-Size' header",
+			zap.Error(err),
+			zap.Strings("header", fileSizeS),
+			zap.String("filename", handler.Filename))
+		http.Error(w, "missing or bad X-File-Size header", http.StatusBadRequest)
+		return
+	}
+
+	// TODO: allow headers to add more metadata (e.g. environment
+	// and function metadata)
+	ss.logger.Info("handling upload",
+		zap.String("filename", handler.Filename))
+	//fileMetadata := make(map[string]interface{})
+	//fileMetadata["filename"] = handler.Filename
+
+	id, err := ss.storageClient.putFile(file, int64(fileSize))
+	if err != nil {
+		ss.logger.Error("error saving uploaded file",
+			zap.Error(err),
+			zap.String("filename", handler.Filename))
+		http.Error(w, "Error saving uploaded file", http.StatusInternalServerError)
+		return
+	}
+
+	// respond with an ID that can be used to retrieve the file
+	ur := &UploadResponse{
+		ID: id,
+	}
+	resp, err := json.Marshal(ur)
+	if err != nil {
+		ss.logger.Error("error marshaling uploaded file response",
+			zap.Error(err),
+			zap.String("filename", handler.Filename))
+		http.Error(w, "Error marshaling response", http.StatusInternalServerError)
+		return
+	}
+	w.Write(resp)
+}
+
+func (ss *StorageService) getIdFromRequest(r *http.Request) (string, error) {
+	values := r.URL.Query()
+	ids, ok := values["id"]
+	if !ok || len(ids) == 0 {
+		return "", errors.New("Missing `id' query param")
+	}
+	return ids[0], nil
+}
+
+func (ss *StorageService) deleteHandler(w http.ResponseWriter, r *http.Request) {
+	// get id from request
+	fileId, err := ss.getIdFromRequest(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	err = ss.storageClient.removeFileByID(fileId)
+	if err != nil {
+		msg := fmt.Sprintf("Error deleting item: %v", err)
+		http.Error(w, msg, http.StatusInternalServerError)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request) {
+	// get id from request
+	fileId, err := ss.getIdFromRequest(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// Get the file (called "item" in stow's jargon), open it,
+	// stream it to response
+	err = ss.storageClient.copyFileToStream(fileId, w)
+	if err != nil {
+		ss.logger.Error("error getting file from storage client", zap.Error(err), zap.String("file_id", fileId))
+		if err == ErrNotFound {
+			http.Error(w, "Error retrieving item: not found", http.StatusNotFound)
+		} else if err == ErrRetrievingItem {
+			http.Error(w, "Error retrieving item", http.StatusBadRequest)
+		} else if err == ErrOpeningItem {
+			http.Error(w, "Error opening item", http.StatusBadRequest)
+		} else if err == ErrWritingFileIntoResponse {
+			http.Error(w, "Error writing response", http.StatusInternalServerError)
+		}
+		return
+	}
+}
+
+func (ss *StorageService) healthHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+func MakeStorageService(logger *zap.Logger, storageClient *StowClient, port int) *StorageService {
+	return &StorageService{
+		logger:        logger.Named("storage_service"),
+		storageClient: storageClient,
+		port:          port,
+	}
+}
+
+func (ss *StorageService) Start(port int) {
+	r := mux.NewRouter()
+	r.HandleFunc("/v1/archive", ss.uploadHandler).Methods("POST")
+	r.HandleFunc("/v1/archive", ss.downloadHandler).Methods("GET")
+	r.HandleFunc("/v1/archive", ss.deleteHandler).Methods("DELETE")
+	r.HandleFunc("/healthz", ss.healthHandler).Methods("GET")
+
+	address := fmt.Sprintf(":%v", port)
+
+	r.Use(utils.LoggingMiddleware(ss.logger))
+	err := http.ListenAndServe(address, &ochttp.Handler{
+		Handler: r,
+		// Propagation: &b3.HTTPFormat{},
+	})
+
+	ss.logger.Fatal("done listening", zap.Error(err))
+}
+
+func RunStorageService(logger *zap.Logger, storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService {
+	// setup a signal handler for SIGTERM
+	utils.SetupStackTraceHandler()
+
+	// create a storage client
+	storageClient, err := MakeStowClient(logger, storageType, storagePath, containerName)
+	if err != nil {
+		logger.Fatal("error creating stowClient", zap.Error(err))
+	}
+
+	// create http handlers
+	storageService := MakeStorageService(logger, storageClient, port)
+	go storageService.Start(port)
+
+	// enablePruner prevents storagesvc unit test from needing to talk to kubernetes
+	if enablePruner {
+		// get the prune interval and start the archive pruner
+		pruneInterval, err := strconv.Atoi(os.Getenv("PRUNE_INTERVAL"))
+		if err != nil {
+			pruneInterval = defaultPruneInterval
+		}
+		pruner, err := MakeArchivePruner(logger, storageClient, time.Duration(pruneInterval))
+		if err != nil {
+			logger.Fatal("error creating archivePruner", zap.Error(err))
+		}
+		go pruner.Start()
+	}
+
+	logger.Info("storage service started")
+	return storageService
+}