CLI to operate archives managed by Storage Service (#2450)

* Add API for listing storage service archives
If id is mentioned we allow user to download specific
archive. If id is not mentioned we list all archives
present with storage service.
This will allow us to build CLI with storage service
and help users to debug storage service.
* Added commands for storagesvc cli and functionalities
* Reusing code and added geturl and download.
* Fixed geturl for localstorage.
* Fixed description of fission archive command
* Added unit tests for function getstorageurl
* Added integration test for archive cli




Co-authored-by: Sanket Sudake <sanketsudake@gmail.com>

pkg/storagesvc/client/client.go

@@ -123,6 +123,33 @@ func (c *Client) GetUrl(id string) string {
 	return fmt.Sprintf("%v/archive?id=%v", c.url, url.PathEscape(id))
 }
 
+func (c *Client) List(ctx context.Context) ([]string, error) {
+	req, err := http.NewRequest(http.MethodGet, c.url+"/archive", nil)
+	if err != nil {
+		return []string{}, err
+	}
+	resp, err := ctxhttp.Do(ctx, c.httpClient, req)
+	if err != nil {
+		return []string{}, err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return []string{}, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		msg := fmt.Sprintf("List error %v", resp.Status)
+		return []string{}, errors.New(msg)
+	}
+
+	var ids []string
+	err = json.Unmarshal(body, &ids)
+	if err != nil {
+		return []string{}, err
+	}
+	return ids, nil
+}
+
 // Download fetches the file identified by ID to the local file path.
 // filePath must not exist.
 func (c *Client) Download(ctx context.Context, id string, filePath string) error {

pkg/storagesvc/client/storagesvc_test.go

@@ -42,20 +42,25 @@ const (
 	minioRegion          = "ap-south-1"
 )
 
-func panicIf(err error) {
+func failTest(t *testing.T, err error) {
 	if err != nil {
-		log.Panicf("Error: %v", err)
+		t.Fatalf("%v", err)
 	}
 }
 
-func MakeTestFile(size int) *os.File {
+func MakeTestFile(size int) (*os.File, error) {
 	f, err := os.CreateTemp("", "storagesvc_test_")
-	panicIf(err)
+
+	if err != nil {
+		return nil, err
+	}
 
 	_, err = f.Write(bytes.Repeat([]byte("."), size))
-	panicIf(err)
 
-	return f
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
 }
 
 func runMinioDockerContainer(pool *dockertest.Pool) *dockertest.Resource {
@@ -120,7 +125,7 @@ func TestS3StorageService(t *testing.T) {
 	config := zap.NewDevelopmentConfig()
 	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
 	logger, err := config.Build()
-	panicIf(err)
+	failTest(t, err)
 
 	log.Println("starting storage svc")
 	os.Setenv("STORAGE_S3_ENDPOINT", endpoint)
@@ -139,27 +144,28 @@ func TestS3StorageService(t *testing.T) {
 	client := MakeClient(fmt.Sprintf("http://localhost:%v/", 8081))
 
 	// generate a test file
-	tmpfile := MakeTestFile(10 * 1024)
+	tmpfile, err := MakeTestFile(10 * 1024)
+	failTest(t, err)
 	defer os.Remove(tmpfile.Name())
 
 	// store it
 	metadata := make(map[string]string)
 	fileID, err := client.Upload(ctx, tmpfile.Name(), &metadata)
-	panicIf(err)
+	failTest(t, err)
 
 	time.Sleep(10 * time.Second)
 
 	// Retrieve file through minioClient
 	reader, err := minioClient.GetObject(bucketName, fileID, minio.GetObjectOptions{})
-	panicIf(err)
+	failTest(t, err)
 	defer reader.Close()
 
 	retThroughMinio, err := os.CreateTemp("", "storagesvc_verify_minio_")
-	panicIf(err)
+	failTest(t, err)
 	defer os.Remove(retThroughMinio.Name())
 
 	stat, err := reader.Stat()
-	panicIf(err)
+	failTest(t, err)
 
 	if _, err := io.CopyN(retThroughMinio, reader, stat.Size); err != nil {
 		log.Fatalln(err)
@@ -167,25 +173,25 @@ func TestS3StorageService(t *testing.T) {
 
 	// Retrieve file through API
 	retThroughAPI, err := os.CreateTemp("", "storagesvc_verify_")
-	panicIf(err)
+	failTest(t, err)
 	os.Remove(retThroughAPI.Name())
 
 	err = client.Download(ctx, fileID, retThroughAPI.Name())
-	panicIf(err)
+	failTest(t, err)
 	defer os.Remove(retThroughAPI.Name())
 
 	// compare contents
 	contentsMinio, err := os.ReadFile(retThroughMinio.Name())
-	panicIf(err)
+	failTest(t, err)
 	contentsAPI, err := os.ReadFile(retThroughAPI.Name())
-	panicIf(err)
+	failTest(t, err)
 	if !bytes.Equal(contentsMinio, contentsAPI) {
 		log.Panic("Contents don't match")
 	}
 
 	// delete uploaded file
 	err = client.Delete(ctx, fileID)
-	panicIf(err)
+	failTest(t, err)
 
 	// make sure download fails
 	err = client.Download(ctx, fileID, "xxx")
@@ -201,7 +207,7 @@ func TestLocalStorageService(t *testing.T) {
 	config := zap.NewDevelopmentConfig()
 	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
 	logger, err := config.Build()
-	panicIf(err)
+	failTest(t, err)
 
 	log.Println("starting storage svc")
 	localPath := fmt.Sprintf("/tmp/%v", testID)
@@ -216,36 +222,45 @@ func TestLocalStorageService(t *testing.T) {
 	client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))
 
 	// generate a test file
-	tmpfile := MakeTestFile(10 * 1024)
+	tmpfile, err := MakeTestFile(10 * 1024)
+	failTest(t, err)
 	defer os.Remove(tmpfile.Name())
 
 	// store it
 	metadata := make(map[string]string)
 	fileID, err := client.Upload(ctx, tmpfile.Name(), &metadata)
-	panicIf(err)
+	failTest(t, err)
+
+	ids, err := client.List(ctx)
+	if err != nil {
+		t.Fatalf("Could not list files: %s", err)
+	}
+	if len(ids) != 1 {
+		t.Fatalf("Expected 1 file, got %v", len(ids))
+	}
 
 	// make a temp file for verification
 	retrievedfile, err := os.CreateTemp("", "storagesvc_verify_")
-	panicIf(err)
+	failTest(t, err)
 	os.Remove(retrievedfile.Name())
 
 	// retrieve uploaded file
 	err = client.Download(ctx, fileID, retrievedfile.Name())
-	panicIf(err)
+	failTest(t, err)
 	defer os.Remove(retrievedfile.Name())
 
 	// compare contents
 	contents1, err := os.ReadFile(tmpfile.Name())
-	panicIf(err)
+	failTest(t, err)
 	contents2, err := os.ReadFile(retrievedfile.Name())
-	panicIf(err)
+	failTest(t, err)
 	if !bytes.Equal(contents1, contents2) {
 		log.Panic("Contents don't match")
 	}
 
 	// delete uploaded file
 	err = client.Delete(ctx, fileID)
-	panicIf(err)
+	failTest(t, err)
 
 	// make sure download fails
 	err = client.Download(ctx, fileID, "xxx")

pkg/storagesvc/storagesvc.go

@@ -67,6 +67,32 @@ func getStorageLocation(config *storageConfig) (stow.Location, error) {
 	return config.storage.dial()
 }
 
+func (ss *StorageService) listItems(w http.ResponseWriter, r *http.Request) {
+	// get all archives on storage
+	// out of them, there may be some just created but not referenced by packages yet.
+	// need to filter them out.
+	archivesInStorage, err := ss.storageClient.getItemIDsWithFilter(ss.storageClient.filterAllItems, false)
+	if err != nil {
+		ss.logger.Error("error getting items from storage", zap.Error(err))
+		return
+	}
+	ss.logger.Debug("archives in storage", zap.Strings("archives", archivesInStorage))
+
+	// respond with the list of items
+	resp, err := json.Marshal(archivesInStorage)
+	if err != nil {
+		http.Error(w, "error marshaling item list", http.StatusInternalServerError)
+		return
+	}
+	_, err = w.Write(resp)
+	if err != nil {
+		ss.logger.Error(
+			"error writing HTTP response",
+			zap.Error(err),
+		)
+	}
+}
+
 // Handle multipart file uploads.
 func (ss *StorageService) uploadHandler(w http.ResponseWriter, r *http.Request) {
 	// handle upload
@@ -203,6 +229,23 @@ func (ss *StorageService) downloadHandler(w http.ResponseWriter, r *http.Request
 	}
 }
 
+func (ss *StorageService) infoHandler(w http.ResponseWriter, r *http.Request) {
+
+	fileId, err := ss.getIdFromRequest(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	itemURL, err := ss.storageClient.getURL(fileId)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+	}
+
+	w.Header().Add("X-FISSION-ARCHIVEURL", itemURL.String())
+
+}
+
 func (ss *StorageService) healthHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
@@ -219,8 +262,10 @@ func (ss *StorageService) Start(ctx context.Context, port int, openTracingEnable
 	r := mux.NewRouter()
 	r.Use(metrics.HTTPMetricMiddleware)
 	r.HandleFunc("/v1/archive", ss.uploadHandler).Methods("POST")
-	r.HandleFunc("/v1/archive", ss.downloadHandler).Methods("GET")
+	r.HandleFunc("/v1/archive", ss.downloadHandler).Queries("id", "{id}").Methods("GET")
+	r.HandleFunc("/v1/archive", ss.listItems).Methods("GET")
 	r.HandleFunc("/v1/archive", ss.deleteHandler).Methods("DELETE")
+	r.HandleFunc("/v1/archive", ss.infoHandler).Methods("HEAD")
 	r.HandleFunc("/healthz", ss.healthHandler).Methods("GET")
 
 	var handler http.Handler

pkg/storagesvc/stowClient.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"mime/multipart"
+	"net/url"
 	"os"
 	"strings"
 	"time"
@@ -182,6 +183,18 @@ func (client *StowClient) removeFileByID(itemID string) error {
 	return client.container.RemoveItem(itemID)
 }
 
+func (client *StowClient) getURL(itemID string) (*url.URL, error) {
+	item, err := client.container.Item(itemID)
+	if err != nil {
+		if err == stow.ErrNotFound {
+			return nil, ErrNotFound
+		} else {
+			return nil, ErrRetrievingItem
+		}
+	}
+	return item.URL(), nil
+}
+
 func (client *StowClient) getFileSize(itemID string) (int64, error) {
 	item, err := client.container.Item(itemID)
 	if err != nil {
@@ -240,3 +253,12 @@ func (client StowClient) filterItemCreatedAMinuteAgo(item stow.Item, currentTime
 	}
 	return false
 }
+
+func (client StowClient) filterAllItems(item stow.Item, _ interface{}) bool {
+	itemLastModTime, _ := item.LastMod()
+	client.logger.Debug("item info",
+		zap.String("item", item.ID()),
+		zap.Time("last_modified_time", itemLastModTime))
+	return false
+
+}