S3 backend for storage service (#1629)

Co-authored-by: Alok Kumar <rajalokan@gmail.com>

pkg/storagesvc/client/storagesvc_test.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
@@ -27,9 +28,17 @@ import (
 	"time"
 
 	"github.com/dchest/uniuri"
-	"go.uber.org/zap"
-
 	"github.com/fission/fission/pkg/storagesvc"
+	"github.com/minio/minio-go/v6"
+	"github.com/ory/dockertest"
+	dc "github.com/ory/dockertest/docker"
+	"go.uber.org/zap"
+)
+
+const (
+	minioAccessKeyID     = "minioadmin"
+	minioSecretAccessKey = "minioadmin"
+	minioRegion          = "ap-south-1"
 )
 
 func panicIf(err error) {
@@ -48,17 +57,152 @@ func MakeTestFile(size int) *os.File {
 	return f
 }
 
-func TestStorageService(t *testing.T) {
-	testId := uniuri.NewLen(8)
-	port := 8080
-	enableArchivePruner := false
+func runMinioDockerContainer(pool *dockertest.Pool) *dockertest.Resource {
+	options := &dockertest.RunOptions{
+		Repository: "minio/minio",
+		Tag:        "latest",
+		Cmd:        []string{"server", "/data"},
+		PortBindings: map[dc.Port][]dc.PortBinding{
+			"9000/tcp": {{HostIP: "", HostPort: "9000"}},
+		},
+	}
+
+	// pulls an image, creates a container based on it and runs it
+	resource, err := pool.RunWithOptions(options)
+	if err != nil {
+		log.Fatalf("Could not start resource: %s", err)
+	}
+	return resource
+}
+
+func startS3StorageService(endpoint, bucketName, subDir string) {
+	// testID := uniuri.NewLen(8)
+	port := 8081
 
 	logger, err := zap.NewDevelopment()
 	panicIf(err)
 
 	log.Println("starting storage svc")
-	_ = storagesvc.RunStorageService(
-		logger, storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner)
+	os.Setenv("STORAGE_S3_ENDPOINT", endpoint)
+	os.Setenv("STORAGE_S3_BUCKET_NAME", bucketName)
+	os.Setenv("STORAGE_S3_SUB_DIR", subDir)
+	os.Setenv("STORAGE_S3_ACCESS_KEY_ID", minioAccessKeyID)
+	os.Setenv("STORAGE_S3_SECRET_ACCESS_KEY", minioSecretAccessKey)
+	os.Setenv("STORAGE_S3_REGION", minioRegion)
+
+	storage := storagesvc.NewS3Storage()
+	_ = storagesvc.Start(logger, storage, port)
+
+}
+
+func TestS3StorageService(t *testing.T) {
+	fmt.Println("Test S3 Storage service")
+	var minioClient *minio.Client
+
+	// Start minio docker container
+	pool, err := dockertest.NewPool("")
+	resource := runMinioDockerContainer(pool)
+
+	endpoint := fmt.Sprintf("localhost:%s", resource.GetPort("9000/tcp"))
+
+	if err := pool.Retry(func() error {
+		minioClient, err = minio.New(endpoint, minioAccessKeyID, minioSecretAccessKey, false)
+		if err != nil {
+			return err
+		}
+
+		// This is to ensure container is up. Just getting minioClient
+		// isn't suffcient to assume container is up.
+		_, err = minioClient.ListBuckets()
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}); err != nil {
+		log.Fatalf("Could not connect to docker: %s", err)
+	}
+
+	defer pool.Purge(resource)
+
+	// Start storagesvc
+	bucketName := "test-s3-service"
+	subDir := "x/y/z"
+	startS3StorageService(endpoint, bucketName, subDir)
+
+	time.Sleep(time.Second)
+	client := MakeClient(fmt.Sprintf("http://localhost:%v/", 8081))
+
+	// generate a test file
+	tmpfile := MakeTestFile(10 * 1024)
+	defer os.Remove(tmpfile.Name())
+
+	// store it
+	metadata := make(map[string]string)
+	ctx := context.Background()
+	fileID, err := client.Upload(ctx, tmpfile.Name(), &metadata)
+	panicIf(err)
+
+	time.Sleep(10 * time.Second)
+
+	// Retrive file trhough minioClient
+	reader, err := minioClient.GetObject(bucketName, fileID, minio.GetObjectOptions{})
+	panicIf(err)
+	defer reader.Close()
+
+	retThroughMinio, err := ioutil.TempFile("", "storagesvc_verify_minio_")
+	panicIf(err)
+	defer os.Remove(retThroughMinio.Name())
+
+	stat, err := reader.Stat()
+	panicIf(err)
+
+	if _, err := io.CopyN(retThroughMinio, reader, stat.Size); err != nil {
+		log.Fatalln(err)
+	}
+
+	// Retrieve file through API
+	retThroughAPI, err := ioutil.TempFile("", "storagesvc_verify_")
+	panicIf(err)
+	os.Remove(retThroughAPI.Name())
+
+	err = client.Download(ctx, fileID, retThroughAPI.Name())
+	panicIf(err)
+	defer os.Remove(retThroughAPI.Name())
+
+	// compare contents
+	contentsMinio, err := ioutil.ReadFile(retThroughMinio.Name())
+	panicIf(err)
+	contentsAPI, err := ioutil.ReadFile(retThroughAPI.Name())
+	panicIf(err)
+	if !bytes.Equal(contentsMinio, contentsAPI) {
+		log.Panic("Contents don't match")
+	}
+
+	// delete uploaded file
+	err = client.Delete(ctx, fileID)
+	panicIf(err)
+
+	// make sure download fails
+	err = client.Download(ctx, fileID, "xxx")
+	if err == nil {
+		log.Panic("Download succeeded but file isn't supposed to exist")
+	}
+
+}
+
+func TestLocalStorageService(t *testing.T) {
+	testID := uniuri.NewLen(8)
+	port := 8080
+
+	logger, err := zap.NewDevelopment()
+	panicIf(err)
+
+	log.Println("starting storage svc")
+	localPath := fmt.Sprintf("/tmp/%v", testID)
+	_ = os.Mkdir(localPath, os.ModePerm)
+	storage := storagesvc.NewLocalStorage(localPath)
+	_ = storagesvc.Start(logger, storage, port)
 
 	time.Sleep(time.Second)
 	client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))
@@ -70,7 +214,7 @@ func TestStorageService(t *testing.T) {
 	// store it
 	metadata := make(map[string]string)
 	ctx := context.Background()
-	fileId, err := client.Upload(ctx, tmpfile.Name(), &metadata)
+	fileID, err := client.Upload(ctx, tmpfile.Name(), &metadata)
 	panicIf(err)
 
 	// make a temp file for verification
@@ -79,7 +223,7 @@ func TestStorageService(t *testing.T) {
 	os.Remove(retrievedfile.Name())
 
 	// retrieve uploaded file
-	err = client.Download(ctx, fileId, retrievedfile.Name())
+	err = client.Download(ctx, fileID, retrievedfile.Name())
 	panicIf(err)
 	defer os.Remove(retrievedfile.Name())
 
@@ -93,15 +237,15 @@ func TestStorageService(t *testing.T) {
 	}
 
 	// delete uploaded file
-	err = client.Delete(ctx, fileId)
+	err = client.Delete(ctx, fileID)
 	panicIf(err)
 
 	// make sure download fails
-	err = client.Download(ctx, fileId, "xxx")
+	err = client.Download(ctx, fileID, "xxx")
 	if err == nil {
 		log.Panic("Download succeeded but file isn't supposed to exist")
 	}
 
-	// cleanup /tmp
-	os.RemoveAll(fmt.Sprintf("/tmp/%v", testId))
+	// // cleanup /tmp
+	os.RemoveAll(fmt.Sprintf("/tmp/%v", testID))
 }