Use mholt/archives instead of mholt/archiver (#3128)
* Use mholt/archives instead of mholt/archiver * Fix validations * Fix iszip function * Fix directory * Add better path sanitization * ensure safe dir is passed * Fix file permissions * Fix config path * Sanitize builder source path --------- Signed-off-by: Sanket Sudake <sanketsudake@gmail.com>
This commit is contained in:
Sanket Sudake
Vendored
BIN
Binary file not shown.
+17
-37
@@ -29,7 +29,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/mholt/archiver/v3"
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/net/context/ctxhttp"
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
@@ -88,21 +87,6 @@ func FindAllGlobs(paths ...string) ([]string, error) {
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func MakeZipArchive(targetName string, globs ...string) (string, error) {
|
||||
files, err := FindAllGlobs(globs...)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// zip up the file list
|
||||
err = archiver.DefaultZip.Archive(files, targetName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return filepath.Abs(targetName)
|
||||
}
|
||||
|
||||
// RemoveZeroBytes remove empty byte(\x00) from input byte slice and return a new byte slice
|
||||
// This function is trying to fix the problem that empty byte will fail os.Openfile
|
||||
// For more information, please visit:
|
||||
@@ -219,15 +203,6 @@ func DownloadUrl(ctx context.Context, httpClient *http.Client, url string, local
|
||||
return nil
|
||||
}
|
||||
|
||||
func IsZip(filename string) (bool, error) {
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
defer f.Close()
|
||||
return archiver.DefaultZip.Match(f)
|
||||
}
|
||||
|
||||
func GetStringValueFromEnv(envVar string) (string, error) {
|
||||
v := os.Getenv(envVar)
|
||||
if v == "" {
|
||||
@@ -310,17 +285,22 @@ func IsOwnerReferencesEnabled() bool {
|
||||
return !disableOwnerReference
|
||||
}
|
||||
|
||||
// ValidateFilePathComponent checks if the filename is valid to prevent directory traversal attacks.
|
||||
func ValidateFilePathComponent(filename string) bool {
|
||||
return len(filename) > 0 && !containsInvalidChars(filename)
|
||||
}
|
||||
|
||||
func containsInvalidChars(filename string) bool {
|
||||
invalidChars := []string{"/", "\\", ".."}
|
||||
for _, char := range invalidChars {
|
||||
if strings.Contains(filename, char) {
|
||||
return true
|
||||
}
|
||||
// SanitizeFilePath checks if the path is valid to prevent directory traversal attacks.
|
||||
func SanitizeFilePath(path string, safedir string) (string, error) {
|
||||
if len(path) == 0 {
|
||||
return "", errors.New("invalid path")
|
||||
}
|
||||
return false
|
||||
if len(safedir) == 0 {
|
||||
return "", errors.New("invalid safe directory")
|
||||
}
|
||||
// get normalized path and check for directory traversal attacks
|
||||
normalizedPath := filepath.Clean(path)
|
||||
if normalizedPath != path {
|
||||
return "", errors.New("invalid path")
|
||||
}
|
||||
// check if the path is under the safe directory
|
||||
if !strings.HasPrefix(normalizedPath, safedir) {
|
||||
return "", fmt.Errorf("path %s is not under the safe directory %s", normalizedPath, safedir)
|
||||
}
|
||||
return normalizedPath, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
)
|
||||
|
||||
func IsZip(ctx context.Context, filename string) (bool, error) {
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
result, err := archives.Zip{}.Match(ctx, filename, f)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if result.ByName || result.ByStream {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func MakeZipArchiveWithGlobs(ctx context.Context, targetName string, globs ...string) (string, error) {
|
||||
globFiles, err := FindAllGlobs(globs...)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(globFiles) == 0 {
|
||||
return "", fmt.Errorf("no files found for globs: %v", globs)
|
||||
}
|
||||
files := make(map[string]string, len(globFiles))
|
||||
for _, file := range globFiles {
|
||||
files[file] = ""
|
||||
}
|
||||
|
||||
archiveFiles, err := archives.FilesFromDisk(ctx, nil, files)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read files from disk: %w", err)
|
||||
}
|
||||
out, err := os.Create(targetName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create archive file: %w", err)
|
||||
}
|
||||
defer out.Close()
|
||||
zip := archives.CompressedArchive{
|
||||
Archival: archives.Zip{},
|
||||
}
|
||||
if err := zip.Archive(ctx, out, archiveFiles); err != nil {
|
||||
return "", fmt.Errorf("failed to create archive: %w", err)
|
||||
}
|
||||
return filepath.Abs(targetName)
|
||||
}
|
||||
|
||||
// Archive zips the contents of directory at src into a new zip file
|
||||
// at dst (note that the contents are zipped, not the directory itself).
|
||||
func Archive(ctx context.Context, src string, dst string) error {
|
||||
srcInfo, err := os.Stat(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get source directory info: %w", err)
|
||||
}
|
||||
if srcInfo.IsDir() {
|
||||
src = src + "/*"
|
||||
}
|
||||
_, err = MakeZipArchiveWithGlobs(ctx, dst, src)
|
||||
return err
|
||||
}
|
||||
|
||||
// Unarchive is a function that unzips a zip file to destination
|
||||
func Unarchive(ctx context.Context, src string, dst string) error {
|
||||
var format archives.Zip
|
||||
file, err := os.Open(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
return format.Extract(ctx, file, func(ctx context.Context, f archives.FileInfo) error {
|
||||
destPath := filepath.Join(dst, f.NameInArchive)
|
||||
// check if the file is a directory
|
||||
if f.IsDir() {
|
||||
return os.MkdirAll(destPath, f.Mode())
|
||||
}
|
||||
|
||||
// check if parent directory exists for the file
|
||||
if err := os.MkdirAll(filepath.Dir(destPath), os.ModeDir|0755); err != nil {
|
||||
return fmt.Errorf("failed to create parent directory: %w", err)
|
||||
}
|
||||
|
||||
// Open file in archive
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file in archive: %w", err)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
// Create file in destination
|
||||
destFile, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file in destination: %w", err)
|
||||
}
|
||||
defer destFile.Close()
|
||||
err = destFile.Chmod(f.Mode())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set file permissions: %w", err)
|
||||
}
|
||||
|
||||
// Copy file contents
|
||||
_, err = io.Copy(destFile, rc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to copy file contents: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsZip(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
setupFn func() string
|
||||
want bool
|
||||
wantErr bool
|
||||
cleanup bool
|
||||
}{
|
||||
{
|
||||
name: "valid zip file",
|
||||
setupFn: func() string {
|
||||
return "testdata/test.zip"
|
||||
},
|
||||
want: true,
|
||||
wantErr: false,
|
||||
cleanup: false,
|
||||
},
|
||||
{
|
||||
name: "non-existent file",
|
||||
setupFn: func() string {
|
||||
return "testdata/non-existent.zip"
|
||||
},
|
||||
want: false,
|
||||
wantErr: false,
|
||||
cleanup: true,
|
||||
},
|
||||
{
|
||||
name: "text file",
|
||||
setupFn: func() string {
|
||||
f, err := os.CreateTemp("", "test-*.txt")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := f.WriteString("hello world"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f.Name()
|
||||
},
|
||||
want: false,
|
||||
wantErr: false,
|
||||
cleanup: true,
|
||||
},
|
||||
{
|
||||
name: "corrupt zip file",
|
||||
setupFn: func() string {
|
||||
f, err := os.CreateTemp("", "corrupt-*.zip")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := f.WriteString("corrupted content"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f.Name()
|
||||
},
|
||||
want: true,
|
||||
wantErr: false,
|
||||
cleanup: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
filename := tt.setupFn()
|
||||
if tt.cleanup && !filepath.IsAbs(filename) {
|
||||
// Cleanup only temp files
|
||||
t.Cleanup(func() {
|
||||
os.Remove(filename)
|
||||
})
|
||||
}
|
||||
|
||||
got, err := IsZip(context.Background(), filename)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("IsZip() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Errorf("IsZip() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveUnarchive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create temp test directories
|
||||
sourceDir, err := os.MkdirTemp("", "zip-test-source-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(sourceDir)
|
||||
|
||||
// Create test files and directories
|
||||
files := map[string][]byte{
|
||||
"file1.txt": []byte("hello world"),
|
||||
"file2.txt": []byte("test content"),
|
||||
"dir1/file3.txt": []byte("nested file"),
|
||||
"dir1/dir2/file4.txt": []byte("deeply nested"),
|
||||
}
|
||||
|
||||
for path, content := range files {
|
||||
fullPath := filepath.Join(sourceDir, path)
|
||||
err := os.MkdirAll(filepath.Dir(fullPath), 0755)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = os.WriteFile(fullPath, content, 0644)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create empty directory
|
||||
emptyDir := filepath.Join(sourceDir, "empty-dir")
|
||||
if err := os.Mkdir(emptyDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
srcPath string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "archive and unarchive directory",
|
||||
srcPath: sourceDir,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "archive and unarchive single file",
|
||||
srcPath: filepath.Join(sourceDir, "file1.txt"),
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Create temp zip file
|
||||
zipFile, err := os.CreateTemp("", "test-*.zip")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zipFile.Close()
|
||||
defer os.Remove(zipFile.Name())
|
||||
|
||||
// Create temp extract directory
|
||||
extractDir, err := os.MkdirTemp("", "zip-test-extract-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(extractDir)
|
||||
|
||||
// Test Archive
|
||||
err = Archive(ctx, tt.srcPath, zipFile.Name())
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Archive() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
// Test is valid zip file
|
||||
isZip, err := IsZip(ctx, zipFile.Name())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !isZip {
|
||||
t.Errorf("Archive() did not create a valid zip file")
|
||||
return
|
||||
}
|
||||
|
||||
// Test Unarchive
|
||||
err = Unarchive(ctx, zipFile.Name(), extractDir)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("Unarchive() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate extracted content
|
||||
err = filepath.Walk(tt.srcPath, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(tt.srcPath, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if relPath == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
extractedPath := filepath.Join(extractDir, relPath)
|
||||
|
||||
extractedInfo, err := os.Stat(extractedPath)
|
||||
if err != nil {
|
||||
t.Errorf("Expected file %s not found in extracted directory", relPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
if info.Mode().Perm() != extractedInfo.Mode().Perm() {
|
||||
t.Errorf("File %s permissions mismatch: got %v, want %v",
|
||||
relPath, extractedInfo.Mode().Perm(), info.Mode().Perm())
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
originalContent, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
extractedContent, err := os.ReadFile(extractedPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if string(originalContent) != string(extractedContent) {
|
||||
t.Errorf("File %s content mismatch", relPath)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveOverwrite(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
// Create initial source directory
|
||||
sourceDir, err := os.MkdirTemp("", "zip-test-source-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(sourceDir)
|
||||
|
||||
// Create initial files
|
||||
initialFiles := map[string][]byte{
|
||||
"old1.txt": []byte("old content 1"),
|
||||
"old2.txt": []byte("old content 2"),
|
||||
}
|
||||
for path, content := range initialFiles {
|
||||
fullPath := filepath.Join(sourceDir, path)
|
||||
if err := os.WriteFile(fullPath, content, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create zip file
|
||||
zipFile, err := os.CreateTemp("", "test-*.zip")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zipFile.Close()
|
||||
defer os.Remove(zipFile.Name())
|
||||
|
||||
// Create initial zip
|
||||
if err := Archive(ctx, sourceDir, zipFile.Name()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create new source directory with different content
|
||||
newSourceDir, err := os.MkdirTemp("", "zip-test-new-source-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(newSourceDir)
|
||||
|
||||
// Create new files
|
||||
newFiles := map[string][]byte{
|
||||
"new1.txt": []byte("new content 1"),
|
||||
"new2.txt": []byte("new content 2"),
|
||||
}
|
||||
for path, content := range newFiles {
|
||||
fullPath := filepath.Join(newSourceDir, path)
|
||||
if err := os.WriteFile(fullPath, content, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Overwrite existing zip
|
||||
if err := Archive(ctx, newSourceDir, zipFile.Name()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create extraction directory
|
||||
extractDir, err := os.MkdirTemp("", "zip-test-extract-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(extractDir)
|
||||
|
||||
// Extract overwritten zip
|
||||
if err := Unarchive(ctx, zipFile.Name(), extractDir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Validate extracted content
|
||||
files, err := os.ReadDir(extractDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify only new files exist
|
||||
expectedFiles := map[string]bool{
|
||||
"new1.txt": false,
|
||||
"new2.txt": false,
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
if _, ok := expectedFiles[f.Name()]; !ok {
|
||||
t.Errorf("Unexpected file found: %s", f.Name())
|
||||
continue
|
||||
}
|
||||
expectedFiles[f.Name()] = true
|
||||
|
||||
// Verify content
|
||||
content, err := os.ReadFile(filepath.Join(extractDir, f.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := newFiles[f.Name()]
|
||||
if string(content) != string(expected) {
|
||||
t.Errorf("File %s content mismatch: got %s, want %s",
|
||||
f.Name(), string(content), string(expected))
|
||||
}
|
||||
}
|
||||
|
||||
// Verify old files do not exist
|
||||
oldFiles := []string{"old1.txt", "old2.txt"}
|
||||
for _, oldFile := range oldFiles {
|
||||
_, err := os.Stat(filepath.Join(extractDir, oldFile))
|
||||
if !os.IsNotExist(err) {
|
||||
t.Errorf("Old file %s should not exist in zip", oldFile)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify all expected files were found
|
||||
for name, found := range expectedFiles {
|
||||
if !found {
|
||||
t.Errorf("Expected file not found: %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user