From 1b9fa5bc3034c9816ceaf3a435c699faa5d8fe1c Mon Sep 17 00:00:00 2001 From: Ta-Ching Chen Date: Fri, 1 Sep 2017 22:50:47 +0800 Subject: [PATCH] Unarchive zip file after fetcher downloads the package (#301) Fetcher now uncompresses archive files. This is triggered by automatically detecting the file format, although maybe we should make that explicit or at least move the detection into the client. --- environments/fetcher/fetcher.go | 45 ++++++++++++++++++++++++++++++--- glide.lock | 38 ++++++++++++++++++++++++++-- glide.yaml | 1 + 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/environments/fetcher/fetcher.go b/environments/fetcher/fetcher.go index b42de2b4..b74777a0 100644 --- a/environments/fetcher/fetcher.go +++ b/environments/fetcher/fetcher.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -13,6 +14,8 @@ import ( "path/filepath" "time" + "github.com/mholt/archiver" + "github.com/satori/go.uuid" "k8s.io/client-go/1.5/kubernetes" "k8s.io/client-go/1.5/pkg/api" @@ -196,12 +199,24 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) { } + // check file type here, if the file is a zip file unarchive it. + if archiver.Zip.Match(tmpPath) { + // unarchive tmp file to a tmp unarchive path + tmpUnarchivePath := filepath.Join(fetcher.sharedVolumePath, uuid.NewV4().String()) + err = fetcher.unarchive(tmpPath, tmpUnarchivePath) + if err != nil { + log.Println(err.Error()) + http.Error(w, err.Error(), 500) + return + } + tmpPath = tmpUnarchivePath + } + // move tmp file to requested filename - err = os.Rename(tmpPath, filepath.Join(fetcher.sharedVolumePath, req.Filename)) + err = fetcher.rename(tmpPath, filepath.Join(fetcher.sharedVolumePath, req.Filename)) if err != nil { - e := fmt.Sprintf("Failed to move file: %v", err) - log.Printf(e) - http.Error(w, e, 500) + log.Println(err.Error()) + http.Error(w, err.Error(), 500) return } @@ -209,3 +224,25 @@ func (fetcher *Fetcher) Handler(w http.ResponseWriter, r *http.Request) { // all done w.WriteHeader(http.StatusOK) } + +func (fetcher *Fetcher) rename(src string, dst string) error { + err := os.Rename(src, dst) + if err != nil { + return errors.New(fmt.Sprintf("Failed to move file: %v", err)) + } + return nil +} + +// archive is a function that zips directory into a zip file +func (fetcher *Fetcher) archive(src string, dst string) error { + return archiver.Zip.Make(dst, []string{src}) +} + +// unarchive is a function that unzips a zip file to destination +func (fetcher *Fetcher) unarchive(src string, dst string) error { + err := archiver.Zip.Open(src, dst) + if err != nil { + return errors.New(fmt.Sprintf("Failed to unzip file: %v", err)) + } + return nil +} diff --git a/glide.lock b/glide.lock index 66d2e07a..142df1e1 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 51b5eda27974aac228f42002cb0a9f89bf2f79d36a574a3694c142c2f83e4042 -updated: 2017-03-24T11:01:00.667996869+08:00 +hash: 2703407b4ae6e28cd146b14ff6b77175a17f0f944f7d75131ac0fa0406793d5b +updated: 2017-08-27T22:38:51.57334855+08:00 imports: - name: github.com/blang/semver version: 60ec3488bfea7cca02b021d106d9911120d25fe9 @@ -37,6 +37,14 @@ imports: - reference - name: github.com/docopt/docopt-go version: 784ddc588536785e7299f7272f39101f7faccc3f +- name: github.com/dsnet/compress + version: 0ae8e136a5df9e3caf6c0f69983608b07438411b + subpackages: + - bzip2 + - bzip2/internal/sais + - internal + - internal/errors + - internal/prefix - name: github.com/emicklei/go-restful version: 3d66f886316ac990eb502aaa89ea38546420b8b7 subpackages: @@ -64,6 +72,8 @@ imports: subpackages: - jsonpb - proto +- name: github.com/golang/snappy + version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/google/gofuzz version: fd52762d25a41827db7ef64c43756fd4b9f7e382 - name: github.com/gorilla/context @@ -72,6 +82,10 @@ imports: version: 3a5767ca75ece5f7f1440b1d16975247f8d8b221 - name: github.com/gorilla/mux version: 392c28fe23e1c45ddba891b0320b3b5df220beea +- name: github.com/howeyc/gopass + version: bf9dde6d0d2c004a008c27aaee91170c786f6db8 +- name: github.com/imdario/mergo + version: e3000cb3d28c72b837601cac94debd91032d19fe - name: github.com/influxdata/influxdb version: b7bb7e8359642b6e071735b50ae41f5eb343fd42 subpackages: @@ -92,6 +106,8 @@ imports: - buffer - jlexer - jwriter +- name: github.com/mholt/archiver + version: fe92d3d85a514d2752ad3b61ec412b598a2754f9 - name: github.com/nats-io/go-nats version: 6949c8e06a246e4177961aab22940b5c411e48f0 subpackages: @@ -107,8 +123,16 @@ imports: - util - name: github.com/nats-io/nuid version: 289cccf02c178dc782430d534e3c1f5b72af807f +- name: github.com/nwaples/rardecode + version: f22b7ef81a0afac9ce1447d37e5ab8e99fbd2f73 - name: github.com/pborman/uuid version: 3d4f2ba23642d3cfd06bd4b54cf03d99d95c0f1b +- name: github.com/pierrec/lz4 + version: 5a3d2245f97fc249850e7802e3c01fad02a1c316 +- name: github.com/pierrec/xxHash + version: a0006b13c722f7f12368c00a3d3c2ae8a999a0c6 + subpackages: + - xxHash32 - name: github.com/PuerkitoBio/purell version: 8a290539e2e8629dbc4e6bad948158f790ec31f4 - name: github.com/PuerkitoBio/urlesc @@ -125,8 +149,18 @@ imports: version: f1f1a805ed361a0e078bb537e4ea78cd37dcf065 subpackages: - codec +- name: github.com/ulikunitz/xz + version: 0c6b41e72360850ca4f98dc341fd999726ea007f + subpackages: + - internal/hash + - internal/xlog + - lzma - name: github.com/urfave/cli version: 0bdeddeeb0f650497d603c4ad7b20cfe685682f6 +- name: golang.org/x/crypto + version: 9477e0b78b9ac3d0b03822fd95422e2fe07627cd + subpackages: + - ssh/terminal - name: golang.org/x/net version: 6acef71eb69611914f7a30939ea9f6e194c78172 subpackages: diff --git a/glide.yaml b/glide.yaml index b047bf53..9d065c62 100644 --- a/glide.yaml +++ b/glide.yaml @@ -39,3 +39,4 @@ import: - package: github.com/nats-io/nats-streaming-server version: ^v0.4.0 - package: github.com/graymeta/stow +- package: github.com/mholt/archiver