From a00895a485c46fa899500be0c7f4dbcc3d779aeb Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 16 Jan 2025 15:34:31 +0000 Subject: [PATCH] add logs and test Signed-off-by: Austin Abro --- src/internal/packager2/layout/create.go | 20 +++- src/internal/packager2/layout/create_test.go | 112 +++++++++++++++++++ src/internal/packager2/layout/package.go | 13 +-- 3 files changed, 129 insertions(+), 16 deletions(-) diff --git a/src/internal/packager2/layout/create.go b/src/internal/packager2/layout/create.go index 8f2429fab8..3fc6cf3298 100644 --- a/src/internal/packager2/layout/create.go +++ b/src/internal/packager2/layout/create.go @@ -921,7 +921,19 @@ func reloadComponentTemplatesInPackage(zarfPackage *v1alpha1.ZarfPackage) error return nil } -func splitFile(srcPath string, chunkSize int) (err error) { +// File will split the file into chunks and remove the original file. +func splitFile(ctx context.Context, srcPath string, chunkSize int) (err error) { + // Remove any existing split files + existingChunks, err := filepath.Glob(srcPath + ".part*") + if err != nil { + return err + } + for _, chunk := range existingChunks { + err := os.Remove(chunk) + if err != nil { + return err + } + } srcFile, err := os.Open(srcPath) if err != nil { return err @@ -953,7 +965,7 @@ func splitFile(srcPath string, chunkSize int) (err error) { // iteration as soon as we're done writing. for { path := fmt.Sprintf("%s.part%03d", srcPath, fileCount+1) - dstFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, helpers.ReadAllWriteUser) + dstFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) if err != nil { return err } @@ -1024,10 +1036,10 @@ func splitFile(srcPath string, chunkSize int) (err error) { return fmt.Errorf("unable to marshal the split package data: %w", err) } path := fmt.Sprintf("%s.part000", srcPath) - if err := os.WriteFile(path, b, helpers.ReadAllWriteUser); err != nil { + if err := os.WriteFile(path, b, 0644); err != nil { return fmt.Errorf("unable to write the file %s: %w", path, err) } progressBar.Successf("Package split across %d files", fileCount+1) - + logger.From(ctx).Info("package split across multiple files", "count", fileCount+1) return nil } diff --git a/src/internal/packager2/layout/create_test.go b/src/internal/packager2/layout/create_test.go index b16d6ace08..044985234d 100644 --- a/src/internal/packager2/layout/create_test.go +++ b/src/internal/packager2/layout/create_test.go @@ -4,6 +4,9 @@ package layout import ( + "context" + "encoding/json" + "fmt" "os" "path/filepath" "testing" @@ -14,8 +17,117 @@ import ( "github.com/zarf-dev/zarf/src/pkg/layout" "github.com/zarf-dev/zarf/src/pkg/lint" "github.com/zarf-dev/zarf/src/test/testutil" + "github.com/zarf-dev/zarf/src/types" ) +func TestSplitFile(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fileSize int + chunkSize int + expectedFileSize int64 + expectedLastFileSize int64 + expectedFileCount int + expectedSha256Sum string + }{ + { + name: "split evenly", + fileSize: 2048, + chunkSize: 16, + expectedFileSize: 16, + expectedLastFileSize: 16, + expectedFileCount: 128, + expectedSha256Sum: "93ecad679eff0df493aaf5d7d615211b0f1d7a919016efb15c98f0b8efb1ba43", + }, + { + name: "split with remainder", + fileSize: 2048, + chunkSize: 10, + expectedFileSize: 10, + expectedLastFileSize: 8, + expectedFileCount: 205, + expectedSha256Sum: "fe8460f4d53d3578aa37191acf55b3db7bbcb706056f4b6b02a0c70f24b0d95a", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + name := "random" + p := filepath.Join(dir, name) + f, err := os.Create(p) + require.NoError(t, err) + b := make([]byte, tt.fileSize) + for i := range tt.fileSize { + b[i] = byte(tt.chunkSize) + } + require.NoError(t, err) + _, err = f.Write(b) + require.NoError(t, err) + err = f.Close() + require.NoError(t, err) + + err = splitFile(context.Background(), p, tt.chunkSize) + require.NoError(t, err) + + _, err = os.Stat(p) + require.ErrorIs(t, err, os.ErrNotExist) + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, tt.expectedFileCount+1) + for i, entry := range entries[1:] { + require.Equal(t, fmt.Sprintf("%s.part%03d", name, i+1), entry.Name()) + + fi, err := entry.Info() + require.NoError(t, err) + if i == len(entries)-2 { + require.Equal(t, tt.expectedLastFileSize, fi.Size()) + } else { + require.Equal(t, tt.expectedFileSize, fi.Size()) + } + } + + b, err = os.ReadFile(filepath.Join(dir, fmt.Sprintf("%s.part000", name))) + require.NoError(t, err) + var data types.ZarfSplitPackageData + err = json.Unmarshal(b, &data) + require.NoError(t, err) + require.Equal(t, tt.expectedFileCount, data.Count) + require.Equal(t, int64(tt.fileSize), data.Bytes) + require.Equal(t, tt.expectedSha256Sum, data.Sha256Sum) + }) + } +} + +func TestSplitDeleteExistingFiles(t *testing.T) { + t.Parallel() + tempDir := t.TempDir() + inputFilename := filepath.Join(tempDir, "testfile.txt") + data := make([]byte, 50) + err := os.WriteFile(inputFilename, data, 0644) + require.NoError(t, err) + // Create many fake split files + for i := range 15 { + _, err := os.Create(fmt.Sprintf("%s.part%03d", inputFilename, i)) + require.NoError(t, err) + } + + chunkSize := 20 + err = splitFile(context.Background(), inputFilename, chunkSize) + require.NoError(t, err) + + // Verify fake split files were deleted + _, err = os.Stat(inputFilename) + require.ErrorIs(t, err, os.ErrNotExist) + entries, err := os.ReadDir(tempDir) + require.NoError(t, err) + // Header file + 3 data files + require.Len(t, entries, 4) +} + func TestCreateSkeleton(t *testing.T) { t.Parallel() diff --git a/src/internal/packager2/layout/package.go b/src/internal/packager2/layout/package.go index 187017895a..c269c331c5 100644 --- a/src/internal/packager2/layout/package.go +++ b/src/internal/packager2/layout/package.go @@ -224,18 +224,7 @@ func (p *PackageLayout) Archive(ctx context.Context, dirPath string, maxPackageS if fi.Size()/int64(chunkSize) > 999 { return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files") } - // Remove any existing split files - existingChunks, err := filepath.Glob(tarballPath + ".part*") - if err != nil { - return err - } - for _, chunk := range existingChunks { - err := os.Remove(chunk) - if err != nil { - return err - } - } - err = splitFile(tarballPath, chunkSize) + err = splitFile(ctx, tarballPath, chunkSize) if err != nil { return fmt.Errorf("unable to split the package archive into multiple files: %w", err) }