Move packages to proejct/pkg to follow go project folder structure convention (#1190)

pkg/storagesvc/README.md

@@ -0,0 +1,26 @@
+# StorageSvc 
+StorageSvc consists of 3 components
+* Storage http request handler
+* StowClient
+* ArchivePruner
+
+## StorageSvc  
+This is the HTTP handler that serves requests to :
+* upload archive into a storage
+* fetch an archive from storage
+* delete archive from storage
+
+## StowClient 
+This is the storage interface layer that interacts with stow package.
+It provides methods to:
+* write a file to storage
+* retrieve a file from storage
+* delete a file from storage
+* get all files on storage
+
+## ArchivePruner
+This acts like a cron job to clean up orphaned archives from storage.
+By default configured to run every hour. The value can be set in Values.yaml to any preferred interval.
+
+
+

pkg/storagesvc/archivePruner.go

@@ -0,0 +1,155 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storagesvc
+
+import (
+	"time"
+
+	"go.uber.org/zap"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fission/fission/pkg/crd"
+)
+
+type ArchivePruner struct {
+	logger        *zap.Logger
+	crdClient     *crd.FissionClient
+	archiveChan   chan (string)
+	stowClient    *StowClient
+	pruneInterval time.Duration
+}
+
+const defaultPruneInterval int = 60 // in minutes
+
+func MakeArchivePruner(logger *zap.Logger, stowClient *StowClient, pruneInterval time.Duration) (*ArchivePruner, error) {
+	crdClient, _, _, err := crd.MakeFissionClient()
+	if err != nil {
+		return nil, err
+	}
+
+	return &ArchivePruner{
+		logger:        logger.Named("archive_pruner"),
+		crdClient:     crdClient,
+		archiveChan:   make(chan string),
+		stowClient:    stowClient,
+		pruneInterval: pruneInterval,
+	}, nil
+}
+
+// pruneArchives listens to archiveChannel for archive ids that need to be deleted
+func (pruner *ArchivePruner) pruneArchives() {
+	pruner.logger.Info("listening to archiveChannel to prune archives")
+	for {
+		select {
+		case archiveID := <-pruner.archiveChan:
+			pruner.logger.Info("sending delete request for archive",
+				zap.String("archive_id", archiveID))
+			if err := pruner.stowClient.removeFileByID(archiveID); err != nil {
+				// logging the error and continuing with other deletions.
+				// hopefully this archive will be deleted in the next iteration.
+				pruner.logger.Error("ignoring error while deleting archive",
+					zap.Error(err),
+					zap.String("archive_id", archiveID))
+			}
+		}
+	}
+}
+
+// insertArchive method just writes the archive ID into the channel.
+func (pruner *ArchivePruner) insertArchive(archiveID string) {
+	pruner.archiveChan <- archiveID
+}
+
+// A user may have deleted pkgs with kubectl or fission cli. That only deletes crd.Package objects from kubernetes
+// and not the archives that are referenced by them, leaving the archives as orphans.
+// getOrphanArchives reaps the orphaned archives.
+func (pruner *ArchivePruner) getOrphanArchives() {
+	pruner.logger.Info("getting orphan archives")
+	archivesRefByPkgs := make([]string, 0)
+	var archiveID string
+
+	// get all pkgs from kubernetes
+	pkgList, err := pruner.crdClient.Packages(metav1.NamespaceAll).List(metav1.ListOptions{})
+	if err != nil {
+		pruner.logger.Error("error getting package list from kubernetes", zap.Error(err))
+		return
+	}
+
+	// extract archives referenced by these pkgs
+	for _, pkg := range pkgList.Items {
+		if pkg.Spec.Deployment.URL != "" {
+			archiveID, err = getQueryParamValue(pkg.Spec.Deployment.URL, "id")
+			if err != nil {
+				pruner.logger.Error("error extracting value of archiveID from deployment url",
+					zap.Error(err),
+					zap.String("url", pkg.Spec.Deployment.URL))
+				return
+			}
+			archivesRefByPkgs = append(archivesRefByPkgs, archiveID)
+		}
+		if pkg.Spec.Source.URL != "" {
+			archiveID, err = getQueryParamValue(pkg.Spec.Source.URL, "id")
+			if err != nil {
+				pruner.logger.Error("error extracting value of archiveID from source url",
+					zap.Error(err),
+					zap.String("url", pkg.Spec.Source.URL))
+				return
+			}
+			archivesRefByPkgs = append(archivesRefByPkgs, archiveID)
+		}
+	}
+
+	pruner.logger.Debug("archives referenced by packagese", zap.Strings("archives", archivesRefByPkgs))
+
+	// get all archives on storage
+	// out of them, there may be some just created but not referenced by packages yet.
+	// need to filter them out.
+	archivesInStorage, err := pruner.stowClient.getItemIDsWithFilter(pruner.stowClient.filterItemCreatedAMinuteAgo, time.Now())
+	if err != nil {
+		pruner.logger.Error("error getting items from storage", zap.Error(err))
+		return
+	}
+	pruner.logger.Debug("archives in storage", zap.Strings("archives", archivesInStorage))
+
+	// difference of the two lists gives us the list of orphan archives. This is just a brute force approach.
+	// need to do something more optimal at scale.
+	orphanedArchives := getDifferenceOfLists(archivesInStorage, archivesRefByPkgs)
+	pruner.logger.Debug("orphan archives", zap.Strings("archives", orphanedArchives))
+
+	// send each orphan archive away for deletion
+	for _, archiveID = range orphanedArchives {
+		pruner.insertArchive(archiveID)
+	}
+
+	return
+}
+
+// Start starts a go routine that listens to a channel for archive IDs that need to deleted.
+// Also wakes up at regular intervals to make a list of archive IDs that need to be reaped
+// and sends them over to the channel for deletion
+func (pruner *ArchivePruner) Start() {
+	ticker := time.NewTicker(pruner.pruneInterval * time.Minute)
+	go pruner.pruneArchives()
+	for {
+		select {
+		case <-ticker.C:
+			// This method fetches unused archive IDs and sends them to archiveChannel for deletion
+			// silencing the errors, hoping they go away in next iteration.
+			pruner.getOrphanArchives()
+		}
+	}
+}

pkg/storagesvc/client/client.go

@@ -0,0 +1,182 @@
+/*
+Copyright 2016 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+
+	"go.opencensus.io/plugin/ochttp"
+	"golang.org/x/net/context/ctxhttp"
+
+	"github.com/fission/fission/pkg/storagesvc"
+)
+
+type (
+	Client struct {
+		url        string
+		httpClient *http.Client
+	}
+)
+
+// Client creates a storage service client.
+func MakeClient(url string) *Client {
+	return &Client{
+		url: strings.TrimSuffix(url, "/") + "/v1",
+		httpClient: &http.Client{
+			Transport: &ochttp.Transport{},
+		},
+	}
+}
+
+// Upload sends the local file pointed to by filePath to the storage
+// service, along with the metadata.  It returns a file ID that can be
+// used to retrieve the file.
+func (c *Client) Upload(ctx context.Context, filePath string, metadata *map[string]string) (string, error) {
+	fi, err := os.Stat(filePath)
+	if err != nil {
+		return "", err
+	}
+	fileSize := fi.Size()
+
+	buf := &bytes.Buffer{}
+	bodyWriter := multipart.NewWriter(buf)
+	fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filePath)
+	if err != nil {
+		return "", err
+	}
+
+	f, err := os.Open(filePath)
+	if err != nil {
+		return "", err
+	}
+
+	_, err = io.Copy(fileWriter, f)
+	if err != nil {
+		return "", err
+	}
+
+	contentType := bodyWriter.FormDataContentType()
+	bodyWriter.Close()
+
+	req, err := http.NewRequest(http.MethodPost, c.url+"/archive", buf)
+	if err != nil {
+		return "", err
+	}
+	req.Header["X-File-Size"] = []string{fmt.Sprintf("%v", fileSize)}
+	req.Header["Content-Type"] = []string{contentType}
+
+	resp, err := ctxhttp.Do(ctx, c.httpClient, req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		msg := fmt.Sprintf("Upload error %v", resp.Status)
+		return "", errors.New(msg)
+	}
+
+	var ur storagesvc.UploadResponse
+	err = json.Unmarshal(body, &ur)
+	if err != nil {
+		return "", err
+	}
+
+	return ur.ID, nil
+}
+
+// GetUrl returns an HTTP URL that can be used to download the file pointed to by ID
+func (c *Client) GetUrl(id string) string {
+	return fmt.Sprintf("%v/archive?id=%v", c.url, url.PathEscape(id))
+}
+
+// Download fetches the file identified by ID to the local file path.
+// filePath must not exist.
+func (c *Client) Download(ctx context.Context, id string, filePath string) error {
+	// url for id
+	url := c.GetUrl(id)
+
+	// quit if file exists
+	_, err := os.Stat(filePath)
+	if err == nil || !os.IsNotExist(err) {
+		return errors.New(fmt.Sprintf("file already exists: %v", filePath))
+	}
+
+	// create
+	f, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	// make request
+	resp, err := ctxhttp.Get(ctx, c.httpClient, url)
+	if err != nil {
+		fmt.Println(err)
+		os.Remove(filePath)
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		msg := fmt.Sprintf("HTTP error %v", resp.StatusCode)
+		os.Remove(filePath)
+		return errors.New(msg)
+	}
+
+	// download and write data
+	_, err = io.Copy(f, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) Delete(ctx context.Context, id string) error {
+	url := c.GetUrl(id)
+
+	req, err := http.NewRequest(http.MethodDelete, url, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := ctxhttp.Do(ctx, c.httpClient, req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(fmt.Sprintf("HTTP error %v", resp.StatusCode))
+	}
+
+	return nil
+}

pkg/storagesvc/client/storagesvc_test.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/dchest/uniuri"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/storagesvc"
+)
+
+func panicIf(err error) {
+	if err != nil {
+		log.Panicf("Error: %v", err)
+	}
+}
+
+func MakeTestFile(size int) *os.File {
+	f, err := ioutil.TempFile("", "storagesvc_test_")
+	panicIf(err)
+
+	_, err = f.Write(bytes.Repeat([]byte("."), size))
+	panicIf(err)
+
+	return f
+}
+
+func TestStorageService(t *testing.T) {
+	testId := uniuri.NewLen(8)
+	port := 8080
+	enableArchivePruner := false
+
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	log.Println("starting storage svc")
+	_ = storagesvc.RunStorageService(
+		logger, storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner)
+
+	time.Sleep(time.Second)
+	client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))
+
+	// generate a test file
+	tmpfile := MakeTestFile(10 * 1024)
+	defer os.Remove(tmpfile.Name())
+
+	// store it
+	metadata := make(map[string]string)
+	ctx := context.Background()
+	fileId, err := client.Upload(ctx, tmpfile.Name(), &metadata)
+	panicIf(err)
+
+	// make a temp file for verification
+	retrievedfile, err := ioutil.TempFile("", "storagesvc_verify_")
+	panicIf(err)
+	os.Remove(retrievedfile.Name())
+
+	// retrieve uploaded file
+	err = client.Download(ctx, fileId, retrievedfile.Name())
+	panicIf(err)
+	defer os.Remove(retrievedfile.Name())
+
+	// compare contents
+	contents1, err := ioutil.ReadFile(tmpfile.Name())
+	panicIf(err)
+	contents2, err := ioutil.ReadFile(retrievedfile.Name())
+	panicIf(err)
+	if !bytes.Equal(contents1, contents2) {
+		log.Panic("Contents don't match")
+	}
+
+	// delete uploaded file
+	err = client.Delete(ctx, fileId)
+	panicIf(err)
+
+	// make sure download fails
+	err = client.Download(ctx, fileId, "xxx")
+	if err == nil {
+		log.Panic("Download succeeded but file isn't supposed to exist")
+	}
+
+	// cleanup /tmp
+	os.RemoveAll(fmt.Sprintf("/tmp/%v", testId))
+}

pkg/storagesvc/storagesvc.go

@@ -0,0 +1,225 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storagesvc
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/gorilla/mux"
+	_ "github.com/graymeta/stow/local"
+	"go.opencensus.io/plugin/ochttp"
+	"go.uber.org/zap"
+
+	"github.com/fission/fission/pkg/utils"
+)
+
+type (
+	StorageService struct {
+		logger        *zap.Logger
+		storageClient *StowClient
+		port          int
+	}
+
+	UploadResponse struct {
+		ID string `json:"id"`
+	}
+)
+
+// Handle multipart file uploads.
+func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request) {
+	// handle upload
+	r.ParseMultipartForm(0)
+	file, handler, err := r.FormFile("uploadfile")
+	if err != nil {
+		http.Error(w, "missing upload file", http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+
+	// stow wants the file size, but that's different from the
+	// content length, the content length being the size of the
+	// encoded file in the HTTP request. So we require an
+	// "X-File-Size" header in bytes.
+
+	fileSizeS, ok := r.Header["X-File-Size"]
+	if !ok {
+		ss.logger.Error("upload is missing the 'X-File-Size' header",
+			zap.String("filename", handler.Filename))
+		http.Error(w, "missing X-File-Size header", http.StatusBadRequest)
+		return
+	}
+
+	fileSize, err := strconv.Atoi(fileSizeS[0])
+	if err != nil {
+		ss.logger.Error("error parsing 'X-File-Size' header",
+			zap.Error(err),
+			zap.Strings("header", fileSizeS),
+			zap.String("filename", handler.Filename))
+		http.Error(w, "missing or bad X-File-Size header", http.StatusBadRequest)
+		return
+	}
+
+	// TODO: allow headers to add more metadata (e.g. environment
+	// and function metadata)
+	ss.logger.Info("handling upload",
+		zap.String("filename", handler.Filename))
+	//fileMetadata := make(map[string]interface{})
+	//fileMetadata["filename"] = handler.Filename
+
+	id, err := ss.storageClient.putFile(file, int64(fileSize))
+	if err != nil {
+		ss.logger.Error("error saving uploaded file",
+			zap.Error(err),
+			zap.String("filename", handler.Filename))
+		http.Error(w, "Error saving uploaded file", http.StatusInternalServerError)
+		return
+	}
+
+	// respond with an ID that can be used to retrieve the file
+	ur := &UploadResponse{
+		ID: id,
+	}
+	resp, err := json.Marshal(ur)
+	if err != nil {
+		ss.logger.Error("error marshaling uploaded file response",
+			zap.Error(err),
+			zap.String("filename", handler.Filename))
+		http.Error(w, "Error marshaling response", http.StatusInternalServerError)
+		return
+	}
+	w.Write(resp)
+}
+
+func (ss *StorageService) getIdFromRequest(r *http.Request) (string, error) {
+	values := r.URL.Query()
+	ids, ok := values["id"]
+	if !ok || len(ids) == 0 {
+		return "", errors.New("Missing `id' query param")
+	}
+	return ids[0], nil
+}
+
+func (ss *StorageService) deleteHandler(w http.ResponseWriter, r *http.Request) {
+	// get id from request
+	fileId, err := ss.getIdFromRequest(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	err = ss.storageClient.removeFileByID(fileId)
+	if err != nil {
+		msg := fmt.Sprintf("Error deleting item: %v", err)
+		http.Error(w, msg, http.StatusInternalServerError)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
+func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request) {
+	// get id from request
+	fileId, err := ss.getIdFromRequest(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// Get the file (called "item" in stow's jargon), open it,
+	// stream it to response
+	err = ss.storageClient.copyFileToStream(fileId, w)
+	if err != nil {
+		ss.logger.Error("error getting file from storage client", zap.Error(err), zap.String("file_id", fileId))
+		if err == ErrNotFound {
+			http.Error(w, "Error retrieving item: not found", http.StatusNotFound)
+		} else if err == ErrRetrievingItem {
+			http.Error(w, "Error retrieving item", http.StatusBadRequest)
+		} else if err == ErrOpeningItem {
+			http.Error(w, "Error opening item", http.StatusBadRequest)
+		} else if err == ErrWritingFileIntoResponse {
+			http.Error(w, "Error writing response", http.StatusInternalServerError)
+		}
+		return
+	}
+}
+
+func (ss *StorageService) healthHandler(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusOK)
+}
+
+func MakeStorageService(logger *zap.Logger, storageClient *StowClient, port int) *StorageService {
+	return &StorageService{
+		logger:        logger.Named("storage_service"),
+		storageClient: storageClient,
+		port:          port,
+	}
+}
+
+func (ss *StorageService) Start(port int) {
+	r := mux.NewRouter()
+	r.HandleFunc("/v1/archive", ss.uploadHandler).Methods("POST")
+	r.HandleFunc("/v1/archive", ss.downloadHandler).Methods("GET")
+	r.HandleFunc("/v1/archive", ss.deleteHandler).Methods("DELETE")
+	r.HandleFunc("/healthz", ss.healthHandler).Methods("GET")
+
+	address := fmt.Sprintf(":%v", port)
+
+	r.Use(utils.LoggingMiddleware(ss.logger))
+	err := http.ListenAndServe(address, &ochttp.Handler{
+		Handler: r,
+		// Propagation: &b3.HTTPFormat{},
+	})
+
+	ss.logger.Fatal("done listening", zap.Error(err))
+}
+
+func RunStorageService(logger *zap.Logger, storageType StorageType, storagePath string, containerName string, port int, enablePruner bool) *StorageService {
+	// setup a signal handler for SIGTERM
+	utils.SetupStackTraceHandler()
+
+	// create a storage client
+	storageClient, err := MakeStowClient(logger, storageType, storagePath, containerName)
+	if err != nil {
+		logger.Fatal("error creating stowClient", zap.Error(err))
+	}
+
+	// create http handlers
+	storageService := MakeStorageService(logger, storageClient, port)
+	go storageService.Start(port)
+
+	// enablePruner prevents storagesvc unit test from needing to talk to kubernetes
+	if enablePruner {
+		// get the prune interval and start the archive pruner
+		pruneInterval, err := strconv.Atoi(os.Getenv("PRUNE_INTERVAL"))
+		if err != nil {
+			pruneInterval = defaultPruneInterval
+		}
+		pruner, err := MakeArchivePruner(logger, storageClient, time.Duration(pruneInterval))
+		if err != nil {
+			logger.Fatal("error creating archivePruner", zap.Error(err))
+		}
+		go pruner.Start()
+	}
+
+	logger.Info("storage service started")
+	return storageService
+}

pkg/storagesvc/stowClient.go

@@ -0,0 +1,206 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storagesvc
+
+import (
+	"io"
+	"mime/multipart"
+	"os"
+	"time"
+
+	"github.com/graymeta/stow"
+	_ "github.com/graymeta/stow/local"
+	"github.com/pkg/errors"
+	"github.com/satori/go.uuid"
+	"go.uber.org/zap"
+)
+
+type (
+	StorageType string
+
+	storageConfig struct {
+		storageType   StorageType
+		localPath     string
+		containerName string
+		// other stuff, such as google or s3 credentials, bucket names etc
+	}
+
+	StowClient struct {
+		logger    *zap.Logger
+		config    *storageConfig
+		location  stow.Location
+		container stow.Container
+	}
+)
+
+const (
+	StorageTypeLocal StorageType = "local"
+	PaginationSize   int         = 10
+)
+
+var (
+	ErrNotFound                = errors.New("not found")
+	ErrRetrievingItem          = errors.New("unable to retrieve item")
+	ErrOpeningItem             = errors.New("unable to open item")
+	ErrWritingFile             = errors.New("unable to write file")
+	ErrWritingFileIntoResponse = errors.New("unable to copy item into http response")
+)
+
+func MakeStowClient(logger *zap.Logger, storageType StorageType, storagePath string, containerName string) (*StowClient, error) {
+	if storageType != StorageTypeLocal {
+		return nil, errors.New("Storage types other than 'local' are not implemented")
+	}
+
+	config := &storageConfig{
+		storageType:   storageType,
+		localPath:     storagePath,
+		containerName: containerName,
+	}
+
+	stowClient := &StowClient{
+		logger: logger.Named("stow_client"),
+		config: config,
+	}
+
+	cfg := stow.ConfigMap{"path": config.localPath}
+	loc, err := stow.Dial("local", cfg)
+	if err != nil {
+		return nil, err
+	}
+	stowClient.location = loc
+
+	con, err := loc.CreateContainer(config.containerName)
+	if os.IsExist(err) {
+		var cons []stow.Container
+		var cursor string
+
+		// use location.Containers to find containers that match the prefix (container name)
+		cons, cursor, err = loc.Containers(config.containerName, stow.CursorStart, 1)
+		if err == nil {
+			if !stow.IsCursorEnd(cursor) {
+				// Should only have one storage container
+				err = errors.New("Found more than one matched storage containers")
+			} else {
+				con = cons[0]
+			}
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	stowClient.container = con
+
+	return stowClient, nil
+}
+
+// putFile writes the file on the storage
+func (client *StowClient) putFile(file multipart.File, fileSize int64) (string, error) {
+	// This is not the item ID (that's returned by Put)
+	// should we just use handler.Filename? what are the constraints here?
+	uploadName := uuid.NewV4().String()
+
+	// save the file to the storage backend
+	item, err := client.container.Put(uploadName, file, int64(fileSize), nil)
+	if err != nil {
+		client.logger.Error("error writing file on storage",
+			zap.Error(err),
+			zap.String("file", uploadName))
+		return "", ErrWritingFile
+	}
+
+	client.logger.Debug("successfully wrote file on storage", zap.String("file", uploadName))
+	return item.ID(), nil
+}
+
+// copyFileToStream gets the file contents into a stream
+func (client *StowClient) copyFileToStream(fileId string, w io.Writer) error {
+	item, err := client.container.Item(fileId)
+	if err != nil {
+		if err == stow.ErrNotFound {
+			return ErrNotFound
+		} else {
+			return ErrRetrievingItem
+		}
+	}
+
+	f, err := item.Open()
+	if err != nil {
+		return ErrOpeningItem
+	}
+	defer f.Close()
+
+	_, err = io.Copy(w, f)
+	if err != nil {
+		return ErrWritingFileIntoResponse
+	}
+
+	client.logger.Debug("successfully wrote file into httpresponse", zap.String("file", fileId))
+	return nil
+}
+
+// removeFileByID deletes the file from storage
+func (client *StowClient) removeFileByID(itemID string) error {
+	return client.container.RemoveItem(itemID)
+}
+
+// filter defines an interface to filter out items from a set of items
+type filter func(stow.Item, interface{}) bool
+
+// This method returns all items in a container, filtering out items based on the filter function passed to it
+func (client *StowClient) getItemIDsWithFilter(filterFunc filter, filterFuncParam interface{}) ([]string, error) {
+	cursor := stow.CursorStart
+	var items []stow.Item
+	var err error
+
+	archiveIDList := make([]string, 0)
+
+	for {
+		items, cursor, err = client.container.Items(stow.NoPrefix, cursor, PaginationSize)
+		if err != nil {
+			errors.Wrap(err, "error getting items from container")
+			return nil, err
+		}
+
+		for _, item := range items {
+			isItemFilterable := filterFunc(item, filterFuncParam)
+			if isItemFilterable {
+				continue
+			}
+			archiveIDList = append(archiveIDList, item.ID())
+		}
+
+		if stow.IsCursorEnd(cursor) {
+			break
+		}
+	}
+
+	return archiveIDList, nil
+}
+
+// filterItemCreatedAMinuteAgo is one type of filter function that filters out items created less than a minute ago.
+// More filter functions can be written if needed, as long as they are of type filter
+func (client StowClient) filterItemCreatedAMinuteAgo(item stow.Item, currentTime interface{}) bool {
+	itemLastModTime, _ := item.LastMod()
+	if currentTime.(time.Time).Sub(itemLastModTime) < 1*time.Minute {
+
+		client.logger.Debug("item created less than a minute ago",
+			zap.String("item", item.ID()),
+			zap.Time("last_modified_time", itemLastModTime))
+		return true
+	}
+	return false
+}

pkg/storagesvc/util.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2017 The Fission Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storagesvc
+
+import (
+	"net/url"
+
+	"github.com/pkg/errors"
+)
+
+func getQueryParamValue(urlString string, queryParam string) (string, error) {
+	url, err := url.Parse(urlString)
+	if err != nil {
+		return "", errors.Wrapf(err, "error parsing URL string %q into URL", urlString)
+	}
+	return url.Query().Get(queryParam), nil
+}
+
+func getDifferenceOfLists(firstList []string, secondList []string) []string {
+	tempMap := make(map[string]int)
+	differenceList := make([]string, 0)
+
+	for _, item := range firstList {
+		tempMap[item] = 1
+	}
+
+	for _, item := range secondList {
+		_, ok := tempMap[item]
+		if ok {
+			delete(tempMap, item)
+		}
+	}
+
+	for k := range tempMap {
+		differenceList = append(differenceList, k)
+	}
+
+	return differenceList
+}