From c380053b1a6d51eef7eab7c31fefe7739235d841 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Tue, 16 Apr 2024 16:52:22 -0700 Subject: [PATCH] feat: add optional sidecar files for metadata This adds the option to store metadata for objects and buckets within a hidden folder at the top level: bucket: .vgw_meta//.meta/ object: .vgw_meta/bucket//.meta/ Example invocation: ./versitygw -a myaccess -s mysecret posix --metadata sidecar /tmp/gw The attributes are stored by name within the hidden directory. --- backend/meta/sidecar.go | 111 +++++++++++++++++++++++++++++++++++++ backend/posix/posix.go | 29 +++++++++- backend/scoutfs/scoutfs.go | 4 +- backend/walk.go | 14 ++++- backend/walk_test.go | 4 +- cmd/versitygw/posix.go | 36 +++++++++--- 6 files changed, 183 insertions(+), 15 deletions(-) create mode 100644 backend/meta/sidecar.go diff --git a/backend/meta/sidecar.go b/backend/meta/sidecar.go new file mode 100644 index 00000000..003acffc --- /dev/null +++ b/backend/meta/sidecar.go @@ -0,0 +1,111 @@ +package meta + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// SideCar is a metadata storer that uses sidecar files to store metadata. +type SideCar struct{} + +const ( + sidecardir = ".vgw_meta" + sidecarmeta = ".meta" +) + +// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket. +func (s SideCar) RetrieveAttribute(f *os.File, bucket, object, attribute string) ([]byte, error) { + metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(sidecardir, bucket, sidecarmeta) + } + attr := filepath.Join(metadir, attribute) + + value, err := os.ReadFile(attr) + if errors.Is(err, os.ErrNotExist) { + return nil, ErrNoSuchKey + } + if err != nil { + return nil, fmt.Errorf("failed to read attribute: %v", err) + } + + return value, nil +} + +// StoreAttribute stores the value of a specific attribute for an object or a bucket. +func (s SideCar) StoreAttribute(f *os.File, bucket, object, attribute string, value []byte) error { + metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(sidecardir, bucket, sidecarmeta) + } + err := os.MkdirAll(metadir, 0777) + if err != nil { + return fmt.Errorf("failed to create metadata directory: %v", err) + } + + attr := filepath.Join(metadir, attribute) + err = os.WriteFile(attr, value, 0666) + if err != nil { + return fmt.Errorf("failed to write attribute: %v", err) + } + + return nil +} + +// DeleteAttribute removes the value of a specific attribute for an object or a bucket. +func (s SideCar) DeleteAttribute(bucket, object, attribute string) error { + metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(sidecardir, bucket, sidecarmeta) + } + attr := filepath.Join(metadir, attribute) + + err := os.Remove(attr) + if errors.Is(err, os.ErrNotExist) { + return ErrNoSuchKey + } + if err != nil { + return fmt.Errorf("failed to remove attribute: %v", err) + } + + return nil +} + +// ListAttributes lists all attributes for an object or a bucket. +func (s SideCar) ListAttributes(bucket, object string) ([]string, error) { + metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(sidecardir, bucket, sidecarmeta) + } + + ents, err := os.ReadDir(metadir) + if errors.Is(err, os.ErrNotExist) { + return []string{}, nil + } + if err != nil { + return nil, fmt.Errorf("failed to list attributes: %v", err) + } + + var attrs []string + for _, ent := range ents { + attrs = append(attrs, ent.Name()) + } + + return attrs, nil +} + +// DeleteAttributes removes all attributes for an object or a bucket. +func (s SideCar) DeleteAttributes(bucket, object string) error { + metadir := filepath.Join(sidecardir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(sidecardir, bucket, sidecarmeta) + } + + err := os.RemoveAll(metadir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove attributes: %v", err) + } + return nil +} diff --git a/backend/posix/posix.go b/backend/posix/posix.go index b2e86096..fcd8a0c2 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -71,6 +71,8 @@ type Posix struct { // newDirPerm is the permission to set on newly created directories newDirPerm fs.FileMode + + skipprefix []string // skip these prefixes when walking } var _ backend.Backend = &Posix{} @@ -91,6 +93,7 @@ const ( bucketLockKey = "bucket-lock" objectRetentionKey = "object-retention" objectLegalHoldKey = "object-legal-hold" + sidecardir = ".vgw_meta" versioningKey = "versioning" deleteMarkerKey = "delete-marker" versionIdKey = "version-id" @@ -107,6 +110,7 @@ type PosixOpts struct { BucketLinks bool VersioningDir string NewDirPerm fs.FileMode + SideCar bool } func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) { @@ -161,6 +165,11 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro fmt.Printf("Bucket versioning enabled with directory: %v\n", verioningdirAbs) + var skipprefx []string + if opts.SideCar { + skipprefx = []string{sidecardir} + } + return &Posix{ meta: meta, rootfd: f, @@ -169,6 +178,7 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro egid: os.Getegid(), chownuid: opts.ChownUID, chowngid: opts.ChownGID, + skipprefix: skipprefx, bucketlinks: opts.BucketLinks, versioningDir: verioningdirAbs, newDirPerm: opts.NewDirPerm, @@ -219,6 +229,12 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput var buckets []s3response.ListAllMyBucketsEntry for _, entry := range entries { + + if containsprefix(entry.Name(), p.skipprefix) { + // skip directories that match the skip prefix + continue + } + fi, err := entry.Info() if err != nil { // skip entries returning errors @@ -295,6 +311,15 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput }, nil } +func containsprefix(a string, strs []string) bool { + for _, s := range strs { + if strings.HasPrefix(a, s) { + return true + } + } + return false +} + func (p *Posix) HeadBucket(_ context.Context, input *s3.HeadBucketInput) (*s3.HeadBucketOutput, error) { if input.Bucket == nil { return nil, s3err.GetAPIError(s3err.ErrInvalidBucketName) @@ -3355,7 +3380,7 @@ func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3 fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, - p.fileToObj(bucket), []string{metaTmpDir}) + p.fileToObj(bucket), []string{metaTmpDir}, p.skipprefix) if err != nil { return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err) } @@ -3487,7 +3512,7 @@ func (p *Posix) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, - p.fileToObj(bucket), []string{metaTmpDir}) + p.fileToObj(bucket), []string{metaTmpDir}, p.skipprefix) if err != nil { return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err) } diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index aca8abfe..17f4e2a8 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -763,7 +763,7 @@ func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) ( fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, - s.fileToObj(bucket), []string{metaTmpDir}) + s.fileToObj(bucket), []string{metaTmpDir}, []string{}) if err != nil { return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err) } @@ -813,7 +813,7 @@ func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Inpu fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys), - s.fileToObj(bucket), []string{metaTmpDir}) + s.fileToObj(bucket), []string{metaTmpDir}, []string{}) if err != nil { return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err) } diff --git a/backend/walk.go b/backend/walk.go index b4365121..5dc5ea65 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -40,7 +40,7 @@ var ErrSkipObj = errors.New("skip this object") // Walk walks the supplied fs.FS and returns results compatible with list // objects responses -func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string) (WalkResults, error) { +func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker string, max int32, getObj GetObjFunc, skipdirs []string, skipprefix []string) (WalkResults, error) { cpmap := make(map[string]struct{}) var objects []s3response.Object @@ -75,6 +75,9 @@ func Walk(ctx context.Context, fileSystem fs.FS, prefix, delimiter, marker strin if contains(d.Name(), skipdirs) { return fs.SkipDir } + if containsprefix(d.Name(), skipprefix) { + return fs.SkipDir + } if pastMax { if len(objects) != 0 { @@ -477,3 +480,12 @@ func WalkVersions(ctx context.Context, fileSystem fs.FS, prefix, delimiter, keyM NextVersionIdMarker: nextVersionIdMarker, }, nil } + +func containsprefix(a string, strs []string) bool { + for _, s := range strs { + if strings.HasPrefix(a, s) { + return true + } + } + return false +} diff --git a/backend/walk_test.go b/backend/walk_test.go index f2f9a492..74377d9b 100644 --- a/backend/walk_test.go +++ b/backend/walk_test.go @@ -224,7 +224,7 @@ func TestWalk(t *testing.T) { for _, tc := range tt.cases { res, err := backend.Walk(context.Background(), tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs, - tt.getobj, []string{}) + tt.getobj, []string{}, []string{}) if err != nil { t.Errorf("tc.name: walk: %v", err) } @@ -363,7 +363,7 @@ func TestWalkStop(t *testing.T) { _, err = backend.Walk(ctx, s, "", "/", "", 1000, func(path string, d fs.DirEntry) (s3response.Object, error) { return s3response.Object{}, nil - }, []string{}) + }, []string{}, []string{}) }() select { diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go index db86c19b..a6b30eea 100644 --- a/cmd/versitygw/posix.go +++ b/cmd/versitygw/posix.go @@ -29,6 +29,7 @@ var ( bucketlinks bool versioningDir string dirPerms uint + metadata string ) func posixCommand() *cli.Command { @@ -79,6 +80,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`, DefaultText: "0755", Value: 0755, }, + &cli.StringFlag{ + Name: "metadata", + Usage: "specify storage option for metadata, default is xattr", + EnvVars: []string{"VGW_META_STORE"}, + Destination: &metadata, + }, }, } } @@ -88,23 +95,36 @@ func runPosix(ctx *cli.Context) error { return fmt.Errorf("no directory provided for operation") } - gwroot := (ctx.Args().Get(0)) - err := meta.XattrMeta{}.Test(gwroot) - if err != nil { - return fmt.Errorf("posix xattr check: %v", err) - } - if dirPerms > math.MaxUint32 { return fmt.Errorf("invalid directory permissions: %d", dirPerms) } - be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{ + gwroot := (ctx.Args().Get(0)) + + opts := posix.PosixOpts{ ChownUID: chownuid, ChownGID: chowngid, BucketLinks: bucketlinks, VersioningDir: versioningDir, NewDirPerm: fs.FileMode(dirPerms), - }) + } + + var ms meta.MetadataStorer + switch metadata { + case "sidecar": + ms = meta.SideCar{} + opts.SideCar = true + case "xattr", "": + ms = meta.XattrMeta{} + err := meta.XattrMeta{}.Test(gwroot) + if err != nil { + return fmt.Errorf("xattr check failed: %v", err) + } + default: + return fmt.Errorf("unknown metadata storage option: %s", metadata) + } + + be, err := posix.New(gwroot, ms, opts) if err != nil { return fmt.Errorf("init posix: %v", err) }