Skip to content

Commit

Permalink
add logs and test
Browse files Browse the repository at this point in the history
Signed-off-by: Austin Abro <[email protected]>
  • Loading branch information
AustinAbro321 committed Jan 16, 2025
1 parent 5698a1e commit a00895a
Show file tree
Hide file tree
Showing 3 changed files with 129 additions and 16 deletions.
20 changes: 16 additions & 4 deletions src/internal/packager2/layout/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -921,7 +921,19 @@ func reloadComponentTemplatesInPackage(zarfPackage *v1alpha1.ZarfPackage) error
return nil
}

func splitFile(srcPath string, chunkSize int) (err error) {
// File will split the file into chunks and remove the original file.
func splitFile(ctx context.Context, srcPath string, chunkSize int) (err error) {
// Remove any existing split files
existingChunks, err := filepath.Glob(srcPath + ".part*")
if err != nil {
return err
}
for _, chunk := range existingChunks {
err := os.Remove(chunk)
if err != nil {
return err
}
}
srcFile, err := os.Open(srcPath)
if err != nil {
return err
Expand Down Expand Up @@ -953,7 +965,7 @@ func splitFile(srcPath string, chunkSize int) (err error) {
// iteration as soon as we're done writing.
for {
path := fmt.Sprintf("%s.part%03d", srcPath, fileCount+1)
dstFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, helpers.ReadAllWriteUser)
dstFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
if err != nil {
return err
}
Expand Down Expand Up @@ -1024,10 +1036,10 @@ func splitFile(srcPath string, chunkSize int) (err error) {
return fmt.Errorf("unable to marshal the split package data: %w", err)
}
path := fmt.Sprintf("%s.part000", srcPath)
if err := os.WriteFile(path, b, helpers.ReadAllWriteUser); err != nil {
if err := os.WriteFile(path, b, 0644); err != nil {
return fmt.Errorf("unable to write the file %s: %w", path, err)
}
progressBar.Successf("Package split across %d files", fileCount+1)

logger.From(ctx).Info("package split across multiple files", "count", fileCount+1)
return nil
}
112 changes: 112 additions & 0 deletions src/internal/packager2/layout/create_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
package layout

import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
Expand All @@ -14,8 +17,117 @@ import (
"github.com/zarf-dev/zarf/src/pkg/layout"
"github.com/zarf-dev/zarf/src/pkg/lint"
"github.com/zarf-dev/zarf/src/test/testutil"
"github.com/zarf-dev/zarf/src/types"
)

func TestSplitFile(t *testing.T) {
t.Parallel()

tests := []struct {
name string
fileSize int
chunkSize int
expectedFileSize int64
expectedLastFileSize int64
expectedFileCount int
expectedSha256Sum string
}{
{
name: "split evenly",
fileSize: 2048,
chunkSize: 16,
expectedFileSize: 16,
expectedLastFileSize: 16,
expectedFileCount: 128,
expectedSha256Sum: "93ecad679eff0df493aaf5d7d615211b0f1d7a919016efb15c98f0b8efb1ba43",
},
{
name: "split with remainder",
fileSize: 2048,
chunkSize: 10,
expectedFileSize: 10,
expectedLastFileSize: 8,
expectedFileCount: 205,
expectedSha256Sum: "fe8460f4d53d3578aa37191acf55b3db7bbcb706056f4b6b02a0c70f24b0d95a",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()

dir := t.TempDir()
name := "random"
p := filepath.Join(dir, name)
f, err := os.Create(p)
require.NoError(t, err)
b := make([]byte, tt.fileSize)
for i := range tt.fileSize {
b[i] = byte(tt.chunkSize)
}
require.NoError(t, err)
_, err = f.Write(b)
require.NoError(t, err)
err = f.Close()
require.NoError(t, err)

err = splitFile(context.Background(), p, tt.chunkSize)
require.NoError(t, err)

_, err = os.Stat(p)
require.ErrorIs(t, err, os.ErrNotExist)
entries, err := os.ReadDir(dir)
require.NoError(t, err)
require.Len(t, entries, tt.expectedFileCount+1)
for i, entry := range entries[1:] {
require.Equal(t, fmt.Sprintf("%s.part%03d", name, i+1), entry.Name())

fi, err := entry.Info()
require.NoError(t, err)
if i == len(entries)-2 {
require.Equal(t, tt.expectedLastFileSize, fi.Size())
} else {
require.Equal(t, tt.expectedFileSize, fi.Size())
}
}

b, err = os.ReadFile(filepath.Join(dir, fmt.Sprintf("%s.part000", name)))
require.NoError(t, err)
var data types.ZarfSplitPackageData
err = json.Unmarshal(b, &data)
require.NoError(t, err)
require.Equal(t, tt.expectedFileCount, data.Count)
require.Equal(t, int64(tt.fileSize), data.Bytes)
require.Equal(t, tt.expectedSha256Sum, data.Sha256Sum)
})
}
}

func TestSplitDeleteExistingFiles(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
inputFilename := filepath.Join(tempDir, "testfile.txt")
data := make([]byte, 50)
err := os.WriteFile(inputFilename, data, 0644)
require.NoError(t, err)
// Create many fake split files
for i := range 15 {
_, err := os.Create(fmt.Sprintf("%s.part%03d", inputFilename, i))
require.NoError(t, err)
}

chunkSize := 20
err = splitFile(context.Background(), inputFilename, chunkSize)
require.NoError(t, err)

// Verify fake split files were deleted
_, err = os.Stat(inputFilename)
require.ErrorIs(t, err, os.ErrNotExist)
entries, err := os.ReadDir(tempDir)
require.NoError(t, err)
// Header file + 3 data files
require.Len(t, entries, 4)
}

func TestCreateSkeleton(t *testing.T) {
t.Parallel()

Expand Down
13 changes: 1 addition & 12 deletions src/internal/packager2/layout/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -224,18 +224,7 @@ func (p *PackageLayout) Archive(ctx context.Context, dirPath string, maxPackageS
if fi.Size()/int64(chunkSize) > 999 {
return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files")
}
// Remove any existing split files
existingChunks, err := filepath.Glob(tarballPath + ".part*")
if err != nil {
return err
}
for _, chunk := range existingChunks {
err := os.Remove(chunk)
if err != nil {
return err
}
}
err = splitFile(tarballPath, chunkSize)
err = splitFile(ctx, tarballPath, chunkSize)
if err != nil {
return fmt.Errorf("unable to split the package archive into multiple files: %w", err)
}
Expand Down

0 comments on commit a00895a

Please sign in to comment.