smruthi2187andTa-Ching Chen
31ba992726
All functions have a pkg reference. This can be a package with either source and a deploy archives, or, a deploy archive. Everytime a function is updated, a new package is created. With archive pruner, the archives that are pointed to by old pkg reference can be deleted from the storage. * High level spec for package pruning. * Skeleton for archive pruning * Adding meat 1 to skeleton. * Adding meat #2. Separated storage service into a httpHandler component and Storage Layer component. * Adding meat #3. getOrphanedArchives in pruner and getItems on stowClient. * Restructured archivePruner methods. * Commiting the day's work. Ready for testing #1. * Fixing compile errors. * Test ready. added a few logs for debugging. * Adding a filter for getItems in stowClient. * After testing. * Added a test for archivePruner. * Adding helm value pruneInterval for testing. * Modified test. * Final test. * Fixing interval from seconds to minutes. * Small change. * Changing debugs to info. * Removing the WIP design * Ran gofmt on all these files. * Fixing prune_interval as string in ENV var. * Addressing all comments, but one. * changing getFile method in stowClient to stream it into a response. * All comments incorporated. * Introducing a new flag for running archivePruner. 1. This flag is disabled for archivePruner to run in unit test. 2. This flag is enabled for archivePruner to run in production. 3. Also disabling test_archive_pruner.sh in this PR. Follow up with next PR to enable it. * Addressing review comments. * Changing the command to generate a file dynamically. * Enabling arching_pruner_test * giving execute permissions to test_archive_pruner.sh * Making changes of positional parameters after recent commit. Change test case permission and removing kubectlPortForward. * Adding debug to see why test_utils.sh passed junk pruneInterval. * shell needs special handling for positional parameters from 10.
102 lines
2.3 KiB
Go
102 lines
2.3 KiB
Go
/*
|
|
Copyright 2017 The Fission Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package client
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/dchest/uniuri"
|
|
|
|
"github.com/fission/fission/storagesvc"
|
|
)
|
|
|
|
func panicIf(err error) {
|
|
if err != nil {
|
|
log.Panicf("Error: %v", err)
|
|
}
|
|
}
|
|
|
|
func MakeTestFile(size int) *os.File {
|
|
f, err := ioutil.TempFile("", "storagesvc_test_")
|
|
panicIf(err)
|
|
|
|
_, err = f.Write(bytes.Repeat([]byte("."), size))
|
|
panicIf(err)
|
|
|
|
return f
|
|
}
|
|
|
|
func TestStorageService(t *testing.T) {
|
|
testId := uniuri.NewLen(8)
|
|
port := 8080
|
|
enableArchivePruner := false
|
|
|
|
log.Println("starting storage svc")
|
|
_ = storagesvc.RunStorageService(
|
|
storagesvc.StorageTypeLocal, "/tmp", testId, port, enableArchivePruner)
|
|
|
|
time.Sleep(time.Second)
|
|
client := MakeClient(fmt.Sprintf("http://localhost:%v/", port))
|
|
|
|
// generate a test file
|
|
tmpfile := MakeTestFile(10 * 1024)
|
|
defer os.Remove(tmpfile.Name())
|
|
|
|
// store it
|
|
metadata := make(map[string]string)
|
|
fileId, err := client.Upload(tmpfile.Name(), &metadata)
|
|
panicIf(err)
|
|
|
|
// make a temp file for verification
|
|
retrievedfile, err := ioutil.TempFile("", "storagesvc_verify_")
|
|
panicIf(err)
|
|
os.Remove(retrievedfile.Name())
|
|
|
|
// retrieve uploaded file
|
|
err = client.Download(fileId, retrievedfile.Name())
|
|
panicIf(err)
|
|
defer os.Remove(retrievedfile.Name())
|
|
|
|
// compare contents
|
|
contents1, err := ioutil.ReadFile(tmpfile.Name())
|
|
panicIf(err)
|
|
contents2, err := ioutil.ReadFile(retrievedfile.Name())
|
|
panicIf(err)
|
|
if !bytes.Equal(contents1, contents2) {
|
|
log.Panic("Contents don't match")
|
|
}
|
|
|
|
// delete uploaded file
|
|
err = client.Delete(fileId)
|
|
panicIf(err)
|
|
|
|
// make sure download fails
|
|
err = client.Download(fileId, "xxx")
|
|
if err == nil {
|
|
log.Panic("Download succeeded but file isn't supposed to exist")
|
|
}
|
|
|
|
// cleanup /tmp
|
|
os.RemoveAll(fmt.Sprintf("/tmp/%v", testId))
|
|
}
|