Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: avoid errors when --max-package-size differs between create runs #3398

Merged
merged 8 commits into from
Jan 27, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions src/internal/packager2/layout/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -921,7 +921,18 @@ func reloadComponentTemplatesInPackage(zarfPackage *v1alpha1.ZarfPackage) error
return nil
}

func splitFile(srcPath string, chunkSize int) (err error) {
func splitFile(ctx context.Context, srcPath string, chunkSize int) (err error) {
// Remove any existing split files
existingChunks, err := filepath.Glob(srcPath + ".part*")
if err != nil {
return err
}
for _, chunk := range existingChunks {
err := os.Remove(chunk)
if err != nil {
return err
}
}
srcFile, err := os.Open(srcPath)
if err != nil {
return err
Expand Down Expand Up @@ -953,7 +964,7 @@ func splitFile(srcPath string, chunkSize int) (err error) {
// iteration as soon as we're done writing.
for {
path := fmt.Sprintf("%s.part%03d", srcPath, fileCount+1)
dstFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, helpers.ReadAllWriteUser)
dstFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
if err != nil {
return err
}
Expand Down Expand Up @@ -1024,10 +1035,10 @@ func splitFile(srcPath string, chunkSize int) (err error) {
return fmt.Errorf("unable to marshal the split package data: %w", err)
}
path := fmt.Sprintf("%s.part000", srcPath)
if err := os.WriteFile(path, b, helpers.ReadAllWriteUser); err != nil {
if err := os.WriteFile(path, b, 0644); err != nil {
return fmt.Errorf("unable to write the file %s: %w", path, err)
}
progressBar.Successf("Package split across %d files", fileCount+1)

logger.From(ctx).Info("package split across files", "count", fileCount+1)
return nil
}
110 changes: 110 additions & 0 deletions src/internal/packager2/layout/create_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
package layout

import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"
Expand All @@ -14,8 +17,115 @@ import (
"github.com/zarf-dev/zarf/src/pkg/layout"
"github.com/zarf-dev/zarf/src/pkg/lint"
"github.com/zarf-dev/zarf/src/test/testutil"
"github.com/zarf-dev/zarf/src/types"
)

func TestSplitFile(t *testing.T) {
t.Parallel()

tests := []struct {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Great test cases 👍

name string
fileSize int
chunkSize int
expectedFileSize int64
expectedLastFileSize int64
expectedFileCount int
expectedSha256Sum string
}{
{
name: "split evenly",
fileSize: 2048,
chunkSize: 16,
expectedFileSize: 16,
expectedLastFileSize: 16,
expectedFileCount: 128,
expectedSha256Sum: "93ecad679eff0df493aaf5d7d615211b0f1d7a919016efb15c98f0b8efb1ba43",
},
{
name: "split with remainder",
fileSize: 2048,
chunkSize: 10,
expectedFileSize: 10,
expectedLastFileSize: 8,
expectedFileCount: 205,
expectedSha256Sum: "fe8460f4d53d3578aa37191acf55b3db7bbcb706056f4b6b02a0c70f24b0d95a",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()

dir := t.TempDir()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nonblocking: this is more of a style thing, but usually I add some small intent comment (example below) on a series of logically similar operations. There's tradeoffs here with just making self-documenting code, so I'm curious where your opinions are at there.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's a reasonable thing to do. I try to break up my tests with indentation and the arrange-act-assert methodology. I don't think comments are always necessary, but when code is non intuitive I do think it helps to provide additional context.

In this case, this test case was actually from packager1 so I just copied it over, going forward I'll try to air on the side of more clarity.

name := "random"
p := filepath.Join(dir, name)
f, err := os.Create(p)
require.NoError(t, err)
b := make([]byte, tt.fileSize)
for i := range tt.fileSize {
b[i] = byte(tt.chunkSize)
}
require.NoError(t, err)
_, err = f.Write(b)
require.NoError(t, err)
err = f.Close()
require.NoError(t, err)

err = splitFile(context.Background(), p, tt.chunkSize)
require.NoError(t, err)

_, err = os.Stat(p)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

e.g.

// Ensure content exists

require.ErrorIs(t, err, os.ErrNotExist)
entries, err := os.ReadDir(dir)
require.NoError(t, err)
require.Len(t, entries, tt.expectedFileCount+1)
for i, entry := range entries[1:] {
require.Equal(t, fmt.Sprintf("%s.part%03d", name, i+1), entry.Name())

fi, err := entry.Info()
require.NoError(t, err)
if i == len(entries)-2 {
require.Equal(t, tt.expectedLastFileSize, fi.Size())
} else {
require.Equal(t, tt.expectedFileSize, fi.Size())
}
}

b, err = os.ReadFile(filepath.Join(dir, fmt.Sprintf("%s.part000", name)))
require.NoError(t, err)
var data types.ZarfSplitPackageData
err = json.Unmarshal(b, &data)
require.NoError(t, err)
require.Equal(t, tt.expectedFileCount, data.Count)
require.Equal(t, int64(tt.fileSize), data.Bytes)
require.Equal(t, tt.expectedSha256Sum, data.Sha256Sum)
})
}
}

func TestSplitDeleteExistingFiles(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
inputFilename := filepath.Join(tempDir, "testfile.txt")
data := make([]byte, 50)
err := os.WriteFile(inputFilename, data, 0644)
require.NoError(t, err)
// Create many fake split files
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

for i := range 15 {
f, err := os.Create(fmt.Sprintf("%s.part%03d", inputFilename, i))
require.NoError(t, err)
require.NoError(t, f.Close())
}

chunkSize := 20
err = splitFile(context.Background(), inputFilename, chunkSize)
require.NoError(t, err)

entries, err := os.ReadDir(tempDir)
require.NoError(t, err)
// Verify only header file + 3 data files remain, and not the 15 test split files
require.Len(t, entries, 4)
}

func TestCreateSkeleton(t *testing.T) {
t.Parallel()

Expand Down
2 changes: 1 addition & 1 deletion src/internal/packager2/layout/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ func (p *PackageLayout) Archive(ctx context.Context, dirPath string, maxPackageS
if fi.Size()/int64(chunkSize) > 999 {
return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files")
}
err := splitFile(tarballPath, chunkSize)
err := splitFile(ctx, tarballPath, chunkSize)
if err != nil {
return fmt.Errorf("unable to split the package archive into multiple files: %w", err)
}
Expand Down
Loading